c3ff14e2c1
Summary: As the first part of the effort of having placing different files on different storage types, this change introduces several things: (1) An experimental interface in FileSystem that specify temperature to a new file created. (2) A test FileSystemWrapper, SimulatedHybridFileSystem, that simulates HDD for a file of "warm" temperature. (3) A simple experimental feature ColumnFamilyOptions.bottommost_temperature. RocksDB would pass this value to FileSystem when creating any bottommost file. (4) A db_bench parameter that applies the (2) and (3) to db_bench. The motivation of the change is to introduce minimal changes that allow us to evolve tiered storage development. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8222 Test Plan: ./db_bench --benchmarks=fillrandom --write_buffer_size=2000000 -max_bytes_for_level_base=20000000 -level_compaction_dynamic_level_bytes --reads=100 -compaction_readahead_size=20000000 --reads=100000 -num=10000000 followed by ./db_bench --benchmarks=readrandom,stats --write_buffer_size=2000000 -max_bytes_for_level_base=20000000 -simulate_hybrid_fs_file=/tmp/warm_file_list -level_compaction_dynamic_level_bytes -compaction_readahead_size=20000000 --reads=500 --threads=16 -use_existing_db --num=10000000 and see results as expected. Reviewed By: ajkr Differential Revision: D28003028 fbshipit-source-id: 4724896d5205730227ba2f17c3fecb11261744ce
90 lines
3.3 KiB
C++
90 lines
3.3 KiB
C++
// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
#pragma once
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
#include <utility>
|
|
|
|
#include "rocksdb/file_system.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
// A FileSystem simulates hybrid file system by ingesting latency and limit
|
|
// IOPs.
|
|
// This class is only used for development purpose and should not be used
|
|
// in production.
|
|
// Right now we ingest 15ms latency and allow 100 requests per second when
|
|
// the file is for warm temperature.
|
|
// When the object is destroyed, the list of warm files are written to a
|
|
// file, which can be used to reopen a FileSystem and still recover the
|
|
// list. This is to allow the information to preserve between db_bench
|
|
// runs.
|
|
class SimulatedHybridFileSystem : public FileSystemWrapper {
|
|
public:
|
|
// metadata_file_name stores metadata of the files, so that it can be
|
|
// loaded after process restarts. If the file doesn't exist, create
|
|
// one. The file is written when the class is destroyed.
|
|
explicit SimulatedHybridFileSystem(const std::shared_ptr<FileSystem>& base,
|
|
const std::string& metadata_file_name);
|
|
|
|
~SimulatedHybridFileSystem() override;
|
|
|
|
public:
|
|
IOStatus NewRandomAccessFile(const std::string& fname,
|
|
const FileOptions& file_opts,
|
|
std::unique_ptr<FSRandomAccessFile>* result,
|
|
IODebugContext* dbg) override;
|
|
IOStatus NewWritableFile(const std::string& fname,
|
|
const FileOptions& file_opts,
|
|
std::unique_ptr<FSWritableFile>* result,
|
|
IODebugContext* dbg) override;
|
|
IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
|
|
IODebugContext* dbg) override;
|
|
|
|
const char* Name() const override { return name_.c_str(); }
|
|
|
|
private:
|
|
// Limit 100 requests per second. Rate limiter is designed to byte but
|
|
// we use it as fixed bytes is one request.
|
|
std::shared_ptr<RateLimiter> rate_limiter_;
|
|
std::mutex mutex_;
|
|
std::unordered_set<std::string> warm_file_set_;
|
|
std::string metadata_file_name_;
|
|
std::string name_;
|
|
};
|
|
|
|
// Simulated random access file that can control IOPs and latency to simulate
|
|
// specific storage media
|
|
class SimulatedHybridRaf : public FSRandomAccessFileWrapper {
|
|
public:
|
|
SimulatedHybridRaf(FSRandomAccessFile* t,
|
|
std::shared_ptr<RateLimiter> rate_limiter,
|
|
Temperature temperature)
|
|
: FSRandomAccessFileWrapper(t),
|
|
rate_limiter_(rate_limiter),
|
|
temperature_(temperature) {}
|
|
|
|
~SimulatedHybridRaf() override {}
|
|
|
|
IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
|
|
Slice* result, char* scratch,
|
|
IODebugContext* dbg) const override;
|
|
|
|
IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs,
|
|
const IOOptions& options, IODebugContext* dbg) override;
|
|
|
|
IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options,
|
|
IODebugContext* dbg) override;
|
|
|
|
private:
|
|
std::shared_ptr<RateLimiter> rate_limiter_;
|
|
Temperature temperature_;
|
|
};
|
|
} // namespace ROCKSDB_NAMESPACE
|
|
|
|
#endif // ROCKSDB_LITE
|