a9d168cfd7
Summary: The current Env/FileSystem API separation has a couple of issues - 1. It requires the user to specify 2 options - ```Options::env``` and ```Options::file_system``` - which means they have to make code changes to benefit from the new APIs. Furthermore, there is a risk of accessing the same APIs in two different ways, through Env in the old way and through FileSystem in the new way. The two may not always match, for example, if env is ```PosixEnv``` and FileSystem is a custom implementation. Any stray RocksDB calls to env will use the ```PosixEnv``` implementation rather than the file_system implementation. 2. There needs to be a simple way for the FileSystem developer to instantiate an Env for backward compatibility purposes. This PR solves the above issues and simplifies the migration in the following ways - 1. Embed a shared_ptr to the ```FileSystem``` in the ```Env```, and remove ```Options::file_system``` as a configurable option. This way, no code changes will be required in application code to benefit from the new API. The default Env constructor uses a ```LegacyFileSystemWrapper``` as the embedded ```FileSystem```. 1a. - This also makes it more robust by ensuring that even if RocksDB has some stray calls to Env APIs rather than FileSystem, they will go through the same object and thus there is no risk of getting out of sync. 2. Provide a ```NewCompositeEnv()``` API that can be used to construct a PosixEnv with a custom FileSystem implementation. This eliminates an indirection to call Env APIs, and relieves the FileSystem developer of the burden of having to implement wrappers for the Env APIs. 3. Add a couple of missing FileSystem APIs - ```SanitizeEnvOptions()``` and ```NewLogger()``` Tests: 1. New unit tests 2. make check and make asan_check Pull Request resolved: https://github.com/facebook/rocksdb/pull/6552 Reviewed By: riversand963 Differential Revision: D20592038 Pulled By: anand1976 fbshipit-source-id: c3801ad4153f96d21d5a3ae26c92ba454d1bf1f7
123 lines
3.9 KiB
C++
123 lines
3.9 KiB
C++
// Copyright (c) 2019-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
#include "env/composite_env_wrapper.h"
|
|
#include "rocksdb/file_system.h"
|
|
#include "options/db_options.h"
|
|
#include "rocksdb/utilities/object_registry.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
FileSystem::FileSystem() {}
|
|
|
|
FileSystem::~FileSystem() {}
|
|
|
|
Status FileSystem::Load(const std::string& value,
|
|
std::shared_ptr<FileSystem>* result) {
|
|
Status s;
|
|
#ifndef ROCKSDB_LITE
|
|
s = ObjectRegistry::NewInstance()->NewSharedObject<FileSystem>(value, result);
|
|
#else
|
|
(void)result;
|
|
s = Status::NotSupported("Cannot load FileSystem in LITE mode: ", value);
|
|
#endif
|
|
return s;
|
|
}
|
|
|
|
IOStatus FileSystem::ReuseWritableFile(const std::string& fname,
|
|
const std::string& old_fname,
|
|
const FileOptions& opts,
|
|
std::unique_ptr<FSWritableFile>* result,
|
|
IODebugContext* dbg) {
|
|
IOStatus s = RenameFile(old_fname, fname, opts.io_options, dbg);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
return NewWritableFile(fname, opts, result, dbg);
|
|
}
|
|
|
|
FileOptions FileSystem::OptimizeForLogRead(
|
|
const FileOptions& file_options) const {
|
|
FileOptions optimized_file_options(file_options);
|
|
optimized_file_options.use_direct_reads = false;
|
|
return optimized_file_options;
|
|
}
|
|
|
|
FileOptions FileSystem::OptimizeForManifestRead(
|
|
const FileOptions& file_options) const {
|
|
FileOptions optimized_file_options(file_options);
|
|
optimized_file_options.use_direct_reads = false;
|
|
return optimized_file_options;
|
|
}
|
|
|
|
FileOptions FileSystem::OptimizeForLogWrite(const FileOptions& file_options,
|
|
const DBOptions& db_options) const {
|
|
FileOptions optimized_file_options(file_options);
|
|
optimized_file_options.bytes_per_sync = db_options.wal_bytes_per_sync;
|
|
optimized_file_options.writable_file_max_buffer_size =
|
|
db_options.writable_file_max_buffer_size;
|
|
return optimized_file_options;
|
|
}
|
|
|
|
FileOptions FileSystem::OptimizeForManifestWrite(
|
|
const FileOptions& file_options) const {
|
|
return file_options;
|
|
}
|
|
|
|
FileOptions FileSystem::OptimizeForCompactionTableWrite(
|
|
const FileOptions& file_options,
|
|
const ImmutableDBOptions& db_options) const {
|
|
FileOptions optimized_file_options(file_options);
|
|
optimized_file_options.use_direct_writes =
|
|
db_options.use_direct_io_for_flush_and_compaction;
|
|
return optimized_file_options;
|
|
}
|
|
|
|
FileOptions FileSystem::OptimizeForCompactionTableRead(
|
|
const FileOptions& file_options,
|
|
const ImmutableDBOptions& db_options) const {
|
|
FileOptions optimized_file_options(file_options);
|
|
optimized_file_options.use_direct_reads = db_options.use_direct_reads;
|
|
return optimized_file_options;
|
|
}
|
|
|
|
Status ReadFileToString(FileSystem* fs, const std::string& fname,
|
|
std::string* data) {
|
|
FileOptions soptions;
|
|
data->clear();
|
|
std::unique_ptr<FSSequentialFile> file;
|
|
Status s = fs->NewSequentialFile(fname, soptions, &file, nullptr);
|
|
if (!s.ok()) {
|
|
return s;
|
|
}
|
|
static const int kBufferSize = 8192;
|
|
char* space = new char[kBufferSize];
|
|
while (true) {
|
|
Slice fragment;
|
|
s = file->Read(kBufferSize, IOOptions(), &fragment, space,
|
|
nullptr);
|
|
if (!s.ok()) {
|
|
break;
|
|
}
|
|
data->append(fragment.data(), fragment.size());
|
|
if (fragment.empty()) {
|
|
break;
|
|
}
|
|
}
|
|
delete[] space;
|
|
return s;
|
|
}
|
|
|
|
#ifdef OS_WIN
|
|
std::shared_ptr<FileSystem> FileSystem::Default() {
|
|
static LegacyFileSystemWrapper default_fs(Env::Default());
|
|
static std::shared_ptr<LegacyFileSystemWrapper> default_fs_ptr(
|
|
&default_fs, [](LegacyFileSystemWrapper*) {});
|
|
return default_fs_ptr;
|
|
}
|
|
#endif
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|