06c3b85b9a
Summary: https://github.com/facebook/rocksdb/pull/6807 extends the logic that identifies and purges obsolete files to blob files handled by RocksDB itself. In order to prevent that from interfering with the current BlobDB code, we need to make sure that `BlobDBOptions::blob_dir` is different from the storage directories used by the base DB. (Note: this is true by default.) The patch adds a check that explicitly disallows this configuration and returns `Status::NotSupported` from `BlobDB::Open` in such cases. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6810 Test Plan: Tested using the BlobDB mode of `db_bench`. Reviewed By: riversand963 Differential Revision: D21412676 Pulled By: ltamasi fbshipit-source-id: 6630cc7481e48c8bf55d59423b25f14d52ffe681
263 lines
10 KiB
C++
263 lines
10 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
#include <functional>
|
|
#include <string>
|
|
#include <vector>
|
|
#include "rocksdb/db.h"
|
|
#include "rocksdb/status.h"
|
|
#include "rocksdb/utilities/stackable_db.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
namespace blob_db {
|
|
|
|
// A wrapped database which puts values of KV pairs in a separate log
|
|
// and store location to the log in the underlying DB.
|
|
//
|
|
// The factory needs to be moved to include/rocksdb/utilities to allow
|
|
// users to use blob DB.
|
|
|
|
struct BlobDBOptions {
|
|
// Name of the directory under the base DB where blobs will be stored. Using
|
|
// a directory where the base DB stores its SST files is not supported.
|
|
// Default is "blob_dir"
|
|
std::string blob_dir = "blob_dir";
|
|
|
|
// whether the blob_dir path is relative or absolute.
|
|
bool path_relative = true;
|
|
|
|
// When max_db_size is reached, evict blob files to free up space
|
|
// instead of returnning NoSpace error on write. Blob files will be
|
|
// evicted from oldest to newest, based on file creation time.
|
|
bool is_fifo = false;
|
|
|
|
// Maximum size of the database (including SST files and blob files).
|
|
//
|
|
// Default: 0 (no limits)
|
|
uint64_t max_db_size = 0;
|
|
|
|
// a new bucket is opened, for ttl_range. So if ttl_range is 600seconds
|
|
// (10 minutes), and the first bucket starts at 1471542000
|
|
// then the blob buckets will be
|
|
// first bucket is 1471542000 - 1471542600
|
|
// second bucket is 1471542600 - 1471543200
|
|
// and so on
|
|
uint64_t ttl_range_secs = 3600;
|
|
|
|
// The smallest value to store in blob log. Values smaller than this threshold
|
|
// will be inlined in base DB together with the key.
|
|
uint64_t min_blob_size = 0;
|
|
|
|
// Allows OS to incrementally sync blob files to disk for every
|
|
// bytes_per_sync bytes written. Users shouldn't rely on it for
|
|
// persistency guarantee.
|
|
uint64_t bytes_per_sync = 512 * 1024;
|
|
|
|
// the target size of each blob file. File will become immutable
|
|
// after it exceeds that size
|
|
uint64_t blob_file_size = 256 * 1024 * 1024;
|
|
|
|
// what compression to use for Blob's
|
|
CompressionType compression = kNoCompression;
|
|
|
|
// If enabled, BlobDB cleans up stale blobs in non-TTL files during compaction
|
|
// by rewriting the remaining live blobs to new files.
|
|
bool enable_garbage_collection = false;
|
|
|
|
// The cutoff in terms of blob file age for garbage collection. Blobs in
|
|
// the oldest N non-TTL blob files will be rewritten when encountered during
|
|
// compaction, where N = garbage_collection_cutoff * number_of_non_TTL_files.
|
|
double garbage_collection_cutoff = 0.25;
|
|
|
|
// Disable all background job. Used for test only.
|
|
bool disable_background_tasks = false;
|
|
|
|
void Dump(Logger* log) const;
|
|
};
|
|
|
|
class BlobDB : public StackableDB {
|
|
public:
|
|
using ROCKSDB_NAMESPACE::StackableDB::Put;
|
|
virtual Status Put(const WriteOptions& options, const Slice& key,
|
|
const Slice& value) override = 0;
|
|
virtual Status Put(const WriteOptions& options,
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
const Slice& value) override {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
return Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family.");
|
|
}
|
|
return Put(options, key, value);
|
|
}
|
|
|
|
using ROCKSDB_NAMESPACE::StackableDB::Delete;
|
|
virtual Status Delete(const WriteOptions& options,
|
|
ColumnFamilyHandle* column_family,
|
|
const Slice& key) override {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
return Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family.");
|
|
}
|
|
assert(db_ != nullptr);
|
|
return db_->Delete(options, column_family, key);
|
|
}
|
|
|
|
virtual Status PutWithTTL(const WriteOptions& options, const Slice& key,
|
|
const Slice& value, uint64_t ttl) = 0;
|
|
virtual Status PutWithTTL(const WriteOptions& options,
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
const Slice& value, uint64_t ttl) {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
return Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family.");
|
|
}
|
|
return PutWithTTL(options, key, value, ttl);
|
|
}
|
|
|
|
// Put with expiration. Key with expiration time equal to
|
|
// std::numeric_limits<uint64_t>::max() means the key don't expire.
|
|
virtual Status PutUntil(const WriteOptions& options, const Slice& key,
|
|
const Slice& value, uint64_t expiration) = 0;
|
|
virtual Status PutUntil(const WriteOptions& options,
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
const Slice& value, uint64_t expiration) {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
return Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family.");
|
|
}
|
|
return PutUntil(options, key, value, expiration);
|
|
}
|
|
|
|
using ROCKSDB_NAMESPACE::StackableDB::Get;
|
|
virtual Status Get(const ReadOptions& options,
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
PinnableSlice* value) override = 0;
|
|
|
|
// Get value and expiration.
|
|
virtual Status Get(const ReadOptions& options,
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
PinnableSlice* value, uint64_t* expiration) = 0;
|
|
virtual Status Get(const ReadOptions& options, const Slice& key,
|
|
PinnableSlice* value, uint64_t* expiration) {
|
|
return Get(options, DefaultColumnFamily(), key, value, expiration);
|
|
}
|
|
|
|
using ROCKSDB_NAMESPACE::StackableDB::MultiGet;
|
|
virtual std::vector<Status> MultiGet(
|
|
const ReadOptions& options,
|
|
const std::vector<Slice>& keys,
|
|
std::vector<std::string>* values) override = 0;
|
|
virtual std::vector<Status> MultiGet(
|
|
const ReadOptions& options,
|
|
const std::vector<ColumnFamilyHandle*>& column_families,
|
|
const std::vector<Slice>& keys,
|
|
std::vector<std::string>* values) override {
|
|
for (auto column_family : column_families) {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
return std::vector<Status>(
|
|
column_families.size(),
|
|
Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family."));
|
|
}
|
|
}
|
|
return MultiGet(options, keys, values);
|
|
}
|
|
virtual void MultiGet(const ReadOptions& /*options*/,
|
|
ColumnFamilyHandle* /*column_family*/,
|
|
const size_t num_keys, const Slice* /*keys*/,
|
|
PinnableSlice* /*values*/, Status* statuses,
|
|
const bool /*sorted_input*/ = false) override {
|
|
for (size_t i = 0; i < num_keys; ++i) {
|
|
statuses[i] = Status::NotSupported(
|
|
"Blob DB doesn't support batched MultiGet");
|
|
}
|
|
}
|
|
|
|
using ROCKSDB_NAMESPACE::StackableDB::SingleDelete;
|
|
virtual Status SingleDelete(const WriteOptions& /*wopts*/,
|
|
ColumnFamilyHandle* /*column_family*/,
|
|
const Slice& /*key*/) override {
|
|
return Status::NotSupported("Not supported operation in blob db.");
|
|
}
|
|
|
|
using ROCKSDB_NAMESPACE::StackableDB::Merge;
|
|
virtual Status Merge(const WriteOptions& /*options*/,
|
|
ColumnFamilyHandle* /*column_family*/,
|
|
const Slice& /*key*/, const Slice& /*value*/) override {
|
|
return Status::NotSupported("Not supported operation in blob db.");
|
|
}
|
|
|
|
virtual Status Write(const WriteOptions& opts,
|
|
WriteBatch* updates) override = 0;
|
|
using ROCKSDB_NAMESPACE::StackableDB::NewIterator;
|
|
virtual Iterator* NewIterator(const ReadOptions& options) override = 0;
|
|
virtual Iterator* NewIterator(const ReadOptions& options,
|
|
ColumnFamilyHandle* column_family) override {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
// Blob DB doesn't support non-default column family.
|
|
return nullptr;
|
|
}
|
|
return NewIterator(options);
|
|
}
|
|
|
|
Status CompactFiles(
|
|
const CompactionOptions& compact_options,
|
|
const std::vector<std::string>& input_file_names, const int output_level,
|
|
const int output_path_id = -1,
|
|
std::vector<std::string>* const output_file_names = nullptr,
|
|
CompactionJobInfo* compaction_job_info = nullptr) override = 0;
|
|
Status CompactFiles(
|
|
const CompactionOptions& compact_options,
|
|
ColumnFamilyHandle* column_family,
|
|
const std::vector<std::string>& input_file_names, const int output_level,
|
|
const int output_path_id = -1,
|
|
std::vector<std::string>* const output_file_names = nullptr,
|
|
CompactionJobInfo* compaction_job_info = nullptr) override {
|
|
if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
|
|
return Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family.");
|
|
}
|
|
|
|
return CompactFiles(compact_options, input_file_names, output_level,
|
|
output_path_id, output_file_names, compaction_job_info);
|
|
}
|
|
|
|
using ROCKSDB_NAMESPACE::StackableDB::Close;
|
|
virtual Status Close() override = 0;
|
|
|
|
// Opening blob db.
|
|
static Status Open(const Options& options, const BlobDBOptions& bdb_options,
|
|
const std::string& dbname, BlobDB** blob_db);
|
|
|
|
static Status Open(const DBOptions& db_options,
|
|
const BlobDBOptions& bdb_options,
|
|
const std::string& dbname,
|
|
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
std::vector<ColumnFamilyHandle*>* handles,
|
|
BlobDB** blob_db);
|
|
|
|
virtual BlobDBOptions GetBlobDBOptions() const = 0;
|
|
|
|
virtual Status SyncBlobFiles() = 0;
|
|
|
|
virtual ~BlobDB() {}
|
|
|
|
protected:
|
|
explicit BlobDB();
|
|
};
|
|
|
|
// Destroy the content of the database.
|
|
Status DestroyBlobDB(const std::string& dbname, const Options& options,
|
|
const BlobDBOptions& bdb_options);
|
|
|
|
} // namespace blob_db
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
#endif // ROCKSDB_LITE
|