05993155ef
Summary: SstFileManager move files that need to be deleted into a trash directory. Deprecate this behaviour and instead add ".trash" extension to files that need to be deleted Closes https://github.com/facebook/rocksdb/pull/2970 Differential Revision: D5976805 Pulled By: IslamAbdelRahman fbshipit-source-id: 27374ece4315610b2792c30ffcd50232d4c9a343
117 lines
3.9 KiB
C++
117 lines
3.9 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
#include <map>
|
|
#include <queue>
|
|
#include <string>
|
|
#include <thread>
|
|
|
|
#include "monitoring/instrumented_mutex.h"
|
|
#include "port/port.h"
|
|
|
|
#include "rocksdb/status.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class Env;
|
|
class Logger;
|
|
class SstFileManagerImpl;
|
|
|
|
// DeleteScheduler allows the DB to enforce a rate limit on file deletion,
|
|
// Instead of deleteing files immediately, files are marked as trash
|
|
// and deleted in a background thread that apply sleep penlty between deletes
|
|
// if they are happening in a rate faster than rate_bytes_per_sec,
|
|
//
|
|
// Rate limiting can be turned off by setting rate_bytes_per_sec = 0, In this
|
|
// case DeleteScheduler will delete files immediately.
|
|
class DeleteScheduler {
|
|
public:
|
|
DeleteScheduler(Env* env, int64_t rate_bytes_per_sec, Logger* info_log,
|
|
SstFileManagerImpl* sst_file_manager);
|
|
|
|
~DeleteScheduler();
|
|
|
|
// Return delete rate limit in bytes per second
|
|
int64_t GetRateBytesPerSecond() { return rate_bytes_per_sec_.load(); }
|
|
|
|
// Set delete rate limit in bytes per second
|
|
void SetRateBytesPerSecond(int64_t bytes_per_sec) {
|
|
return rate_bytes_per_sec_.store(bytes_per_sec);
|
|
}
|
|
|
|
// Mark file as trash directory and schedule it's deletion
|
|
Status DeleteFile(const std::string& fname);
|
|
|
|
// Wait for all files being deleteing in the background to finish or for
|
|
// destructor to be called.
|
|
void WaitForEmptyTrash();
|
|
|
|
// Return a map containing errors that happened in BackgroundEmptyTrash
|
|
// file_path => error status
|
|
std::map<std::string, Status> GetBackgroundErrors();
|
|
|
|
uint64_t GetTotalTrashSize() { return total_trash_size_.load(); }
|
|
|
|
void TEST_SetMaxTrashDBRatio(double r) {
|
|
assert(r >= 0);
|
|
max_trash_db_ratio_ = r;
|
|
}
|
|
|
|
static const std::string kTrashExtension;
|
|
static bool IsTrashFile(const std::string& file_path);
|
|
|
|
// Check if there are any .trash filse in path, and schedule their deletion
|
|
// Or delete immediately if sst_file_manager is nullptr
|
|
static Status CleanupDirectory(Env* env, SstFileManagerImpl* sfm,
|
|
const std::string& path);
|
|
|
|
private:
|
|
Status MarkAsTrash(const std::string& file_path, std::string* path_in_trash);
|
|
|
|
Status DeleteTrashFile(const std::string& path_in_trash,
|
|
uint64_t* deleted_bytes);
|
|
|
|
void BackgroundEmptyTrash();
|
|
|
|
Env* env_;
|
|
// total size of trash files
|
|
std::atomic<uint64_t> total_trash_size_;
|
|
// Maximum number of bytes that should be deleted per second
|
|
std::atomic<int64_t> rate_bytes_per_sec_;
|
|
// Mutex to protect queue_, pending_files_, bg_errors_, closing_
|
|
InstrumentedMutex mu_;
|
|
// Queue of trash files that need to be deleted
|
|
std::queue<std::string> queue_;
|
|
// Number of trash files that are waiting to be deleted
|
|
int32_t pending_files_;
|
|
// Errors that happened in BackgroundEmptyTrash (file_path => error)
|
|
std::map<std::string, Status> bg_errors_;
|
|
// Set to true in ~DeleteScheduler() to force BackgroundEmptyTrash to stop
|
|
bool closing_;
|
|
// Condition variable signaled in these conditions
|
|
// - pending_files_ value change from 0 => 1
|
|
// - pending_files_ value change from 1 => 0
|
|
// - closing_ value is set to true
|
|
InstrumentedCondVar cv_;
|
|
// Background thread running BackgroundEmptyTrash
|
|
std::unique_ptr<port::Thread> bg_thread_;
|
|
// Mutex to protect threads from file name conflicts
|
|
InstrumentedMutex file_move_mu_;
|
|
Logger* info_log_;
|
|
SstFileManagerImpl* sst_file_manager_;
|
|
// If the trash size constitutes for more than 25% of the total DB size
|
|
// we will start deleting new files passed to DeleteScheduler immediately
|
|
double max_trash_db_ratio_ = 0.25;
|
|
static const uint64_t kMicrosInSecond = 1000 * 1000LL;
|
|
};
|
|
|
|
} // namespace rocksdb
|
|
|
|
#endif // ROCKSDB_LITE
|