b864bc9b5b
Summary: Improving blob db FIFO eviction with the following changes, * Change blob_dir_size to max_db_size. Take into account SST file size when computing DB size. * FIFO now only take into account live sst files and live blob files. It is normal for disk usage to go over max_db_size because there are obsolete sst files and blob files pending deletion. * FIFO eviction now also evict TTL blob files that's still open. It doesn't evict non-TTL blob files. * If FIFO is triggered, it will pass an expiration and the current sequence number to compaction filter. Compaction filter will then filter inlined keys to evict those with an earlier expiration and smaller sequence number. So call LSM FIFO. * Compaction filter also filter those blob indexes where corresponding blob file is gone. * Add an event listener to listen compaction/flush event and update sst file size. * Implement DB::Close() to make sure base db, as well as event listener and compaction filter, destruct before blob db. * More blob db statistics around FIFO. * Fix some locking issue when accessing a blob file. Closes https://github.com/facebook/rocksdb/pull/3556 Differential Revision: D7139328 Pulled By: yiwu-arbug fbshipit-source-id: ea5edb07b33dfceacb2682f4789bea61de28bbfa
113 lines
3.9 KiB
C++
113 lines
3.9 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
#ifndef __STDC_FORMAT_MACROS
|
|
#define __STDC_FORMAT_MACROS
|
|
#endif
|
|
|
|
#include "utilities/blob_db/blob_db.h"
|
|
|
|
#include <inttypes.h>
|
|
#include "utilities/blob_db/blob_db_impl.h"
|
|
|
|
namespace rocksdb {
|
|
namespace blob_db {
|
|
|
|
Status BlobDB::Open(const Options& options, const BlobDBOptions& bdb_options,
|
|
const std::string& dbname, BlobDB** blob_db) {
|
|
*blob_db = nullptr;
|
|
DBOptions db_options(options);
|
|
ColumnFamilyOptions cf_options(options);
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
column_families.push_back(
|
|
ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
|
|
std::vector<ColumnFamilyHandle*> handles;
|
|
Status s = BlobDB::Open(db_options, bdb_options, dbname, column_families,
|
|
&handles, blob_db);
|
|
if (s.ok()) {
|
|
assert(handles.size() == 1);
|
|
// i can delete the handle since DBImpl is always holding a reference to
|
|
// default column family
|
|
delete handles[0];
|
|
}
|
|
return s;
|
|
}
|
|
|
|
Status BlobDB::Open(const DBOptions& db_options,
|
|
const BlobDBOptions& bdb_options, const std::string& dbname,
|
|
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
std::vector<ColumnFamilyHandle*>* handles,
|
|
BlobDB** blob_db) {
|
|
if (column_families.size() != 1 ||
|
|
column_families[0].name != kDefaultColumnFamilyName) {
|
|
return Status::NotSupported(
|
|
"Blob DB doesn't support non-default column family.");
|
|
}
|
|
|
|
BlobDBImpl* blob_db_impl = new BlobDBImpl(dbname, bdb_options, db_options,
|
|
column_families[0].options);
|
|
Status s = blob_db_impl->Open(handles);
|
|
if (s.ok()) {
|
|
*blob_db = static_cast<BlobDB*>(blob_db_impl);
|
|
} else {
|
|
delete blob_db_impl;
|
|
*blob_db = nullptr;
|
|
}
|
|
return s;
|
|
}
|
|
|
|
BlobDB::BlobDB() : StackableDB(nullptr) {}
|
|
|
|
void BlobDBOptions::Dump(Logger* log) const {
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.blob_dir: %s",
|
|
blob_dir.c_str());
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.path_relative: %d",
|
|
path_relative);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.is_fifo: %d",
|
|
is_fifo);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.max_db_size: %" PRIu64,
|
|
max_db_size);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.ttl_range_secs: %" PRIu32,
|
|
ttl_range_secs);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.min_blob_size: %" PRIu64,
|
|
min_blob_size);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.bytes_per_sync: %" PRIu64,
|
|
bytes_per_sync);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.blob_file_size: %" PRIu64,
|
|
blob_file_size);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.ttl_extractor: %p",
|
|
ttl_extractor.get());
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.compression: %d",
|
|
static_cast<int>(compression));
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.enable_garbage_collection: %d",
|
|
enable_garbage_collection);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.garbage_collection_interval_secs: %" PRIu64,
|
|
garbage_collection_interval_secs);
|
|
ROCKS_LOG_HEADER(
|
|
log, "BlobDBOptions.garbage_collection_deletion_size_threshold: %lf",
|
|
garbage_collection_deletion_size_threshold);
|
|
ROCKS_LOG_HEADER(
|
|
log, " BlobDBOptions.disable_background_tasks: %d",
|
|
disable_background_tasks);
|
|
}
|
|
|
|
} // namespace blob_db
|
|
} // namespace rocksdb
|
|
#endif
|