From f72c834eab08f6e72780d8fead6039acfe565f44 Mon Sep 17 00:00:00 2001 From: mrambacher Date: Tue, 2 Nov 2021 09:06:02 -0700 Subject: [PATCH] Make FileSystem a Customizable Class (#8649) Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/8649 Reviewed By: zhichao-cao Differential Revision: D32036059 Pulled By: mrambacher fbshipit-source-id: 4f1e7557ecac52eb849b83ae02b8d7d232112295 --- HISTORY.md | 3 + db/db_basic_test.cc | 3 + env/env.cc | 3 +- env/env_chroot.cc | 127 ++++++++------- env/env_chroot.h | 32 +++- env/env_encryption.cc | 32 +++- env/env_encryption_ctr.h | 6 + env/env_test.cc | 171 ++++++++++++++++++-- env/file_system.cc | 112 +++++++++++-- env/file_system_tracer.h | 3 + env/fs_posix.cc | 18 ++- env/fs_readonly.h | 3 + env/fs_remap.h | 8 + env/mock_env.cc | 147 ++++------------- env/mock_env.h | 106 ++++++++++++ file/prefetch_test.cc | 3 + include/rocksdb/env_encryption.h | 11 ++ include/rocksdb/file_system.h | 20 ++- options/customizable_test.cc | 37 +++++ port/win/env_win.h | 5 +- test_util/testutil.h | 3 + utilities/env_timed.cc | 266 ++++++++++++++++--------------- utilities/env_timed.h | 97 +++++++++++ utilities/fault_injection_fs.h | 3 +- 24 files changed, 874 insertions(+), 345 deletions(-) create mode 100644 utilities/env_timed.h diff --git a/HISTORY.md b/HISTORY.md index 810d481e5..af28ae9b6 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -15,6 +15,9 @@ ### Public Interface Change * When options.ttl is used with leveled compaction with compactinon priority kMinOverlappingRatio, files exceeding half of TTL value will be prioritized more, so that by the time TTL is reached, fewer extra compactions will be scheduled to clear them up. At the same time, when compacting files with data older than half of TTL, output files may be cut off based on those files' boundaries, in order for the early TTL compaction to work properly. +### Public API change +* Made FileSystem extend the Customizable class and added a CreateFromString method. Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method. + ## 6.26.0 (2021-10-20) ### Bug Fixes * Fixes a bug in directed IO mode when calling MultiGet() for blobs in the same blob file. The bug is caused by not sorting the blob read requests by file offsets. diff --git a/db/db_basic_test.cc b/db/db_basic_test.cc index 3571ce0ec..dd788f49d 100644 --- a/db/db_basic_test.cc +++ b/db/db_basic_test.cc @@ -3296,6 +3296,9 @@ class DeadlineFS : public FileSystemWrapper { ignore_deadline_(false), error_on_delay_(error_on_delay) {} + static const char* kClassName() { return "DeadlineFileSystem"; } + const char* Name() const override { return kClassName(); } + IOStatus NewRandomAccessFile(const std::string& fname, const FileOptions& opts, std::unique_ptr* result, diff --git a/env/env.cc b/env/env.cc index 144c898a9..0bf8e5edc 100644 --- a/env/env.cc +++ b/env/env.cc @@ -324,7 +324,8 @@ class LegacyFileSystemWrapper : public FileSystem { explicit LegacyFileSystemWrapper(Env* t) : target_(t) {} ~LegacyFileSystemWrapper() override {} - const char* Name() const override { return "Legacy File System"; } + static const char* kClassName() { return "LegacyFileSystem"; } + const char* Name() const override { return kClassName(); } // Return the target to which this Env forwards all calls Env* target() const { return target_; } diff --git a/env/env_chroot.cc b/env/env_chroot.cc index ff47049e0..e7cd04031 100644 --- a/env/env_chroot.cc +++ b/env/env_chroot.cc @@ -13,20 +13,35 @@ #include "env/composite_env_wrapper.h" #include "env/fs_remap.h" +#include "rocksdb/utilities/options_type.h" #include "util/string_util.h" // errnoStr namespace ROCKSDB_NAMESPACE { namespace { -class ChrootFileSystem : public RemapFileSystem { - public: - ChrootFileSystem(const std::shared_ptr& base, - const std::string& chroot_dir) - : RemapFileSystem(base) { +static std::unordered_map chroot_fs_type_info = { + {"chroot_dir", {0, OptionType::kString}}}; +} // namespace +ChrootFileSystem::ChrootFileSystem(const std::shared_ptr& base, + const std::string& chroot_dir) + : RemapFileSystem(base), chroot_dir_(chroot_dir) { + RegisterOptions("chroot_dir", &chroot_dir_, &chroot_fs_type_info); +} + +Status ChrootFileSystem::PrepareOptions(const ConfigOptions& options) { + Status s = FileSystemWrapper::PrepareOptions(options); + if (!s.ok()) { + return s; + } else if (chroot_dir_.empty()) { + s = Status::InvalidArgument("ChRootFileSystem requires a chroot dir"); + } else { + s = target_->FileExists(chroot_dir_, IOOptions(), nullptr); + } + if (s.ok()) { #if defined(OS_AIX) char resolvedName[PATH_MAX]; - char* real_chroot_dir = realpath(chroot_dir.c_str(), resolvedName); + char* real_chroot_dir = realpath(chroot_dir_.c_str(), resolvedName); #else - char* real_chroot_dir = realpath(chroot_dir.c_str(), nullptr); + char* real_chroot_dir = realpath(chroot_dir_.c_str(), nullptr); #endif // chroot_dir must exist so realpath() returns non-nullptr. assert(real_chroot_dir != nullptr); @@ -35,32 +50,32 @@ class ChrootFileSystem : public RemapFileSystem { free(real_chroot_dir); #endif } + return s; +} - const char* Name() const override { return "ChrootFS"; } +IOStatus ChrootFileSystem::GetTestDirectory(const IOOptions& options, + std::string* path, + IODebugContext* dbg) { + // Adapted from PosixEnv's implementation since it doesn't provide a way to + // create directory in the chroot. + char buf[256]; + snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast(geteuid())); + *path = buf; - IOStatus GetTestDirectory(const IOOptions& options, std::string* path, - IODebugContext* dbg) override { - // Adapted from PosixEnv's implementation since it doesn't provide a way to - // create directory in the chroot. - char buf[256]; - snprintf(buf, sizeof(buf), "/rocksdbtest-%d", static_cast(geteuid())); - *path = buf; + // Directory may already exist, so ignore return + return CreateDirIfMissing(*path, options, dbg); +} - // Directory may already exist, so ignore return - return CreateDirIfMissing(*path, options, dbg); - } - - protected: // Returns status and expanded absolute path including the chroot directory. // Checks whether the provided path breaks out of the chroot. If it returns // non-OK status, the returned path should not be used. - std::pair EncodePath( - const std::string& path) override { - if (path.empty() || path[0] != '/') { - return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""}; - } - std::pair res; - res.second = chroot_dir_ + path; +std::pair ChrootFileSystem::EncodePath( + const std::string& path) { + if (path.empty() || path[0] != '/') { + return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""}; + } + std::pair res; + res.second = chroot_dir_ + path; #if defined(OS_AIX) char resolvedName[PATH_MAX]; char* normalized_path = realpath(res.second.c_str(), resolvedName); @@ -81,47 +96,51 @@ class ChrootFileSystem : public RemapFileSystem { free(normalized_path); #endif return res; - } +} // Similar to EncodePath() except assumes the basename in the path hasn't been // created yet. - std::pair EncodePathWithNewBasename( - const std::string& path) override { - if (path.empty() || path[0] != '/') { - return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""}; - } - // Basename may be followed by trailing slashes - size_t final_idx = path.find_last_not_of('/'); - if (final_idx == std::string::npos) { - // It's only slashes so no basename to extract - return EncodePath(path); - } - - // Pull off the basename temporarily since realname(3) (used by - // EncodePath()) requires a path that exists - size_t base_sep = path.rfind('/', final_idx); - auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1)); - status_and_enc_path.second.append(path.substr(base_sep + 1)); - return status_and_enc_path; +std::pair ChrootFileSystem::EncodePathWithNewBasename( + const std::string& path) { + if (path.empty() || path[0] != '/') { + return {IOStatus::InvalidArgument(path, "Not an absolute path"), ""}; + } + // Basename may be followed by trailing slashes + size_t final_idx = path.find_last_not_of('/'); + if (final_idx == std::string::npos) { + // It's only slashes so no basename to extract + return EncodePath(path); } - private: - std::string chroot_dir_; -}; -} // namespace + // Pull off the basename temporarily since realname(3) (used by + // EncodePath()) requires a path that exists + size_t base_sep = path.rfind('/', final_idx); + auto status_and_enc_path = EncodePath(path.substr(0, base_sep + 1)); + status_and_enc_path.second.append(path.substr(base_sep + 1)); + return status_and_enc_path; +} std::shared_ptr NewChrootFileSystem( const std::shared_ptr& base, const std::string& chroot_dir) { - return std::make_shared(base, chroot_dir); + auto chroot_fs = std::make_shared(base, chroot_dir); + Status s = chroot_fs->PrepareOptions(ConfigOptions()); + if (s.ok()) { + return chroot_fs; + } else { + return nullptr; + } } Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir) { if (!base_env->FileExists(chroot_dir).ok()) { return nullptr; } - std::shared_ptr chroot_fs = - NewChrootFileSystem(base_env->GetFileSystem(), chroot_dir); - return new CompositeEnvWrapper(base_env, chroot_fs); + auto chroot_fs = NewChrootFileSystem(base_env->GetFileSystem(), chroot_dir); + if (chroot_fs != nullptr) { + return new CompositeEnvWrapper(base_env, chroot_fs); + } else { + return nullptr; + } } } // namespace ROCKSDB_NAMESPACE diff --git a/env/env_chroot.h b/env/env_chroot.h index fb5b70c44..9e5b9a1e9 100644 --- a/env/env_chroot.h +++ b/env/env_chroot.h @@ -9,9 +9,37 @@ #include -#include "rocksdb/env.h" +#include "env/fs_remap.h" +#include "rocksdb/file_system.h" namespace ROCKSDB_NAMESPACE { +class ChrootFileSystem : public RemapFileSystem { + public: + ChrootFileSystem(const std::shared_ptr& base, + const std::string& chroot_dir); + + static const char* kClassName() { return "ChrootFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override; + + Status PrepareOptions(const ConfigOptions& options) override; + + protected: + // Returns status and expanded absolute path including the chroot directory. + // Checks whether the provided path breaks out of the chroot. If it returns + // non-OK status, the returned path should not be used. + std::pair EncodePath(const std::string& path) override; + + // Similar to EncodePath() except assumes the basename in the path hasn't been + // created yet. + std::pair EncodePathWithNewBasename( + const std::string& path) override; + + private: + std::string chroot_dir_; +}; // Returns an Env that translates paths such that the root directory appears to // be chroot_dir. chroot_dir should refer to an existing directory. @@ -19,6 +47,8 @@ namespace ROCKSDB_NAMESPACE { // This class has not been fully analyzed for providing strong security // guarantees. Env* NewChrootEnv(Env* base_env, const std::string& chroot_dir); +std::shared_ptr NewChrootFileSystem( + const std::shared_ptr& base, const std::string& chroot_dir); } // namespace ROCKSDB_NAMESPACE diff --git a/env/env_encryption.cc b/env/env_encryption.cc index accc07c9a..97c7439c4 100644 --- a/env/env_encryption.cc +++ b/env/env_encryption.cc @@ -438,11 +438,20 @@ IOStatus EncryptedRandomRWFile::Close(const IOOptions& options, } namespace { +static std::unordered_map encrypted_fs_type_info = + { + {"provider", + OptionTypeInfo::AsCustomSharedPtr( + 0 /* No offset, whole struct*/, OptionVerificationType::kByName, + OptionTypeFlags::kNone)}, +}; // EncryptedFileSystemImpl implements an FileSystemWrapper that adds encryption // to files stored on disk. class EncryptedFileSystemImpl : public EncryptedFileSystem { public: - const char* Name() const override { return "EncryptedFS"; } + const char* Name() const override { + return EncryptedFileSystem::kClassName(); + } // Returns the raw encryption provider that should be used to write the input // encrypted file. If there is no such provider, NotFound is returned. IOStatus GetWritableProvider(const std::string& /*fname*/, @@ -664,6 +673,7 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem { const std::shared_ptr& provider) : EncryptedFileSystem(base) { provider_ = provider; + RegisterOptions("EncryptionProvider", &provider_, &encrypted_fs_type_info); } Status AddCipher(const std::string& descriptor, const char* cipher, @@ -910,10 +920,28 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem { }; } // namespace +Status NewEncryptedFileSystemImpl( + const std::shared_ptr& base, + const std::shared_ptr& provider, + std::unique_ptr* result) { + result->reset(new EncryptedFileSystemImpl(base, provider)); + return Status::OK(); +} + std::shared_ptr NewEncryptedFS( const std::shared_ptr& base, const std::shared_ptr& provider) { - return std::make_shared(base, provider); + std::unique_ptr efs; + Status s = NewEncryptedFileSystemImpl(base, provider, &efs); + if (s.ok()) { + s = efs->PrepareOptions(ConfigOptions()); + } + if (s.ok()) { + std::shared_ptr result(efs.release()); + return result; + } else { + return nullptr; + } } // Returns an Env that encrypts data when stored on disk and decrypts data when // read from disk. diff --git a/env/env_encryption_ctr.h b/env/env_encryption_ctr.h index f034708c2..405bc6269 100644 --- a/env/env_encryption_ctr.h +++ b/env/env_encryption_ctr.h @@ -105,6 +105,12 @@ class CTREncryptionProvider : public EncryptionProvider { uint64_t initialCounter, const Slice& iv, const Slice& prefix, std::unique_ptr* result); }; + +Status NewEncryptedFileSystemImpl( + const std::shared_ptr& base_fs, + const std::shared_ptr& provider, + std::unique_ptr* fs); + } // namespace ROCKSDB_NAMESPACE #endif // !defined(ROCKSDB_LITE) diff --git a/env/env_test.cc b/env/env_test.cc index 9fd088dad..f03abb640 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -39,6 +39,7 @@ #include "env/emulated_clock.h" #include "env/env_chroot.h" #include "env/env_encryption_ctr.h" +#include "env/fs_readonly.h" #include "env/unique_id_gen.h" #include "logging/log_buffer.h" #include "logging/logging.h" @@ -59,6 +60,7 @@ #include "util/mutexlock.h" #include "util/random.h" #include "util/string_util.h" +#include "utilities/env_timed.h" #include "utilities/fault_injection_env.h" #include "utilities/fault_injection_fs.h" @@ -2233,14 +2235,17 @@ INSTANTIATE_TEST_CASE_P(DefaultEnvWithDirectIO, EnvPosixTestWithParam, #endif // !defined(ROCKSDB_LITE) #if !defined(ROCKSDB_LITE) && !defined(OS_WIN) -static std::unique_ptr chroot_env( - NewChrootEnv(Env::Default(), test::TmpDir(Env::Default()))); -INSTANTIATE_TEST_CASE_P( - ChrootEnvWithoutDirectIO, EnvPosixTestWithParam, - ::testing::Values(std::pair(chroot_env.get(), false))); -INSTANTIATE_TEST_CASE_P( - ChrootEnvWithDirectIO, EnvPosixTestWithParam, - ::testing::Values(std::pair(chroot_env.get(), true))); +static Env* GetChrootEnv() { + static std::unique_ptr chroot_env( + NewChrootEnv(Env::Default(), test::TmpDir(Env::Default()))); + return chroot_env.get(); +} +INSTANTIATE_TEST_CASE_P(ChrootEnvWithoutDirectIO, EnvPosixTestWithParam, + ::testing::Values(std::pair(GetChrootEnv(), + false))); +INSTANTIATE_TEST_CASE_P(ChrootEnvWithDirectIO, EnvPosixTestWithParam, + ::testing::Values(std::pair(GetChrootEnv(), + true))); #endif // !defined(ROCKSDB_LITE) && !defined(OS_WIN) class EnvFSTestWithParam @@ -2492,7 +2497,6 @@ TEST_F(CreateEnvTest, CreateMockSystemClock) { guard->reset(new MockSystemClock(nullptr)); return guard->get(); }); - ASSERT_OK(SystemClock::CreateFromString( config_options_, EmulatedSystemClock::kClassName(), &mock)); ASSERT_NE(mock, nullptr); @@ -2518,6 +2522,154 @@ TEST_F(CreateEnvTest, CreateMockSystemClock) { ASSERT_OK(SystemClock::CreateFromString( config_options_, EmulatedSystemClock::kClassName(), &mock)); } + +TEST_F(CreateEnvTest, CreateReadOnlyFileSystem) { + std::shared_ptr fs, copy; + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, ReadOnlyFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + std::string("id=") + ReadOnlyFileSystem::kClassName() + + "; target=" + TimedFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + opts_str = fs->ToString(config_options_); + ASSERT_STREQ(fs->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + +TEST_F(CreateEnvTest, CreateTimedFileSystem) { + std::shared_ptr fs, copy; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, + TimedFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + std::string("id=") + TimedFileSystem::kClassName() + + "; target=" + ReadOnlyFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + opts_str = fs->ToString(config_options_); + ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), ReadOnlyFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} +#ifndef OS_WIN +TEST_F(CreateEnvTest, CreateChrootFileSystem) { + std::shared_ptr fs, copy; + auto tmp_dir = test::TmpDir(Env::Default()); + // The Chroot FileSystem has a required "chroot_dir" option. + ASSERT_NOK(FileSystem::CreateFromString(config_options_, + ChrootFileSystem::kClassName(), &fs)); + + // ChrootFileSystem fails with an invalid directory + ASSERT_NOK(FileSystem::CreateFromString( + config_options_, + std::string("chroot_dir=/No/Such/Directory; id=") + + ChrootFileSystem::kClassName(), + &fs)); + std::string chroot_opts = std::string("chroot_dir=") + tmp_dir + + std::string("; id=") + + ChrootFileSystem::kClassName(); + + // Create a valid ChrootFileSystem with an inner Default + ASSERT_OK(FileSystem::CreateFromString(config_options_, chroot_opts, &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), ChrootFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + // Create a valid ChrootFileSystem with an inner TimedFileSystem + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + chroot_opts + "; target=" + TimedFileSystem::kClassName(), &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), ChrootFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + opts_str = fs->ToString(config_options_); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + + // Create a TimedFileSystem with an inner ChrootFileSystem + ASSERT_OK(FileSystem::CreateFromString( + config_options_, + "target={" + chroot_opts + "}; id=" + TimedFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), TimedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), ChrootFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + opts_str = fs->ToString(config_options_); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} +#endif // OS_WIN + +TEST_F(CreateEnvTest, CreateEncryptedFileSystem) { + std::shared_ptr fs, copy; + + std::string base_opts = + std::string("provider=1://test; id=") + EncryptedFileSystem::kClassName(); + // The EncryptedFileSystem requires a "provider" option. + ASSERT_NOK(FileSystem::CreateFromString( + config_options_, EncryptedFileSystem::kClassName(), &fs)); + + ASSERT_OK(FileSystem::CreateFromString(config_options_, base_opts, &fs)); + + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), EncryptedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner(), FileSystem::Default().get()); + std::string opts_str = fs->ToString(config_options_); + std::string mismatch; + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); + ASSERT_OK(FileSystem::CreateFromString( + config_options_, base_opts + "; target=" + TimedFileSystem::kClassName(), + &fs)); + ASSERT_NE(fs, nullptr); + ASSERT_STREQ(fs->Name(), EncryptedFileSystem::kClassName()); + ASSERT_NE(fs->Inner(), nullptr); + ASSERT_STREQ(fs->Inner()->Name(), TimedFileSystem::kClassName()); + ASSERT_EQ(fs->Inner()->Inner(), FileSystem::Default().get()); + opts_str = fs->ToString(config_options_); + ASSERT_OK(FileSystem::CreateFromString(config_options_, opts_str, ©)); + ASSERT_TRUE(fs->AreEquivalent(config_options_, copy.get(), &mismatch)); +} + #endif // ROCKSDB_LITE namespace { @@ -2742,7 +2894,6 @@ TEST_F(EnvTest, FailureToCreateLockFile) { // Clean up ASSERT_OK(DestroyDir(env, dir)); } - } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/env/file_system.cc b/env/file_system.cc index a6a2f3388..90edc4b27 100644 --- a/env/file_system.cc +++ b/env/file_system.cc @@ -6,9 +6,17 @@ #include "rocksdb/file_system.h" #include "env/composite_env_wrapper.h" +#include "env/env_chroot.h" +#include "env/env_encryption_ctr.h" +#include "env/fs_readonly.h" +#include "env/mock_env.h" #include "options/db_options.h" #include "rocksdb/convenience.h" +#include "rocksdb/utilities/customizable_util.h" #include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" +#include "util/string_util.h" +#include "utilities/env_timed.h" namespace ROCKSDB_NAMESPACE { @@ -21,19 +29,63 @@ Status FileSystem::Load(const std::string& value, return CreateFromString(ConfigOptions(), value, result); } +#ifndef ROCKSDB_LITE +static int RegisterBuiltinFileSystems(ObjectLibrary& library, + const std::string& /*arg*/) { + library.Register( + TimedFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new TimedFileSystem(nullptr)); + return guard->get(); + }); + library.Register( + ReadOnlyFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new ReadOnlyFileSystem(nullptr)); + return guard->get(); + }); + library.Register( + EncryptedFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* errmsg) { + Status s = NewEncryptedFileSystemImpl(nullptr, nullptr, guard); + if (!s.ok()) { + *errmsg = s.ToString(); + } + return guard->get(); + }); +#ifndef OS_WIN + library.Register( + ChrootFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new ChrootFileSystem(nullptr, "")); + return guard->get(); + }); +#endif // OS_WIN + size_t num_types; + return static_cast(library.GetFactoryCount(&num_types)); +} +#endif // ROCKSDB_LITE + Status FileSystem::CreateFromString(const ConfigOptions& config_options, const std::string& value, std::shared_ptr* result) { - Status s; + auto default_fs = FileSystem::Default(); + if (default_fs->IsInstanceOf(value)) { + *result = default_fs; + return Status::OK(); + } else { #ifndef ROCKSDB_LITE - (void)config_options; - s = ObjectRegistry::NewInstance()->NewSharedObject(value, result); -#else - (void)config_options; - (void)result; - s = Status::NotSupported("Cannot load FileSystem in LITE mode", value); -#endif - return s; + static std::once_flag once; + std::call_once(once, [&]() { + RegisterBuiltinFileSystems(*(ObjectLibrary::Default().get()), ""); + }); +#endif // ROCKSDB_LITE + return LoadSharedObject(config_options, value, nullptr, result); + } } IOStatus FileSystem::ReuseWritableFile(const std::string& fname, @@ -147,4 +199,46 @@ IOStatus ReadFileToString(FileSystem* fs, const std::string& fname, return s; } +namespace { +static std::unordered_map fs_wrapper_type_info = { +#ifndef ROCKSDB_LITE + {"target", + OptionTypeInfo::AsCustomSharedPtr( + 0, OptionVerificationType::kByName, OptionTypeFlags::kDontSerialize)}, +#endif // ROCKSDB_LITE +}; +} // namespace +FileSystemWrapper::FileSystemWrapper(const std::shared_ptr& t) + : target_(t) { + RegisterOptions("", &target_, &fs_wrapper_type_info); +} + +Status FileSystemWrapper::PrepareOptions(const ConfigOptions& options) { + if (target_ == nullptr) { + target_ = FileSystem::Default(); + } + return FileSystem::PrepareOptions(options); +} + +#ifndef ROCKSDB_LITE +std::string FileSystemWrapper::SerializeOptions( + const ConfigOptions& config_options, const std::string& header) const { + auto parent = FileSystem::SerializeOptions(config_options, ""); + if (config_options.IsShallow() || target_ == nullptr || + target_->IsInstanceOf(FileSystem::kDefaultName())) { + return parent; + } else { + std::string result = header; + if (!StartsWith(parent, OptionTypeInfo::kIdPropName())) { + result.append(OptionTypeInfo::kIdPropName()).append("="); + } + result.append(parent); + if (!EndsWith(result, config_options.delimiter)) { + result.append(config_options.delimiter); + } + result.append("target=").append(target_->ToString(config_options)); + return result; + } +} +#endif // ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/env/file_system_tracer.h b/env/file_system_tracer.h index 0c0b918c8..c8ee160f9 100644 --- a/env/file_system_tracer.h +++ b/env/file_system_tracer.h @@ -27,6 +27,9 @@ class FileSystemTracingWrapper : public FileSystemWrapper { ~FileSystemTracingWrapper() override {} + static const char* kClassName() { return "FileSystemTracing"; } + const char* Name() const override { return kClassName(); } + IOStatus NewSequentialFile(const std::string& fname, const FileOptions& file_opts, std::unique_ptr* result, diff --git a/env/fs_posix.cc b/env/fs_posix.cc index d321a746f..52264ce7c 100644 --- a/env/fs_posix.cc +++ b/env/fs_posix.cc @@ -141,7 +141,9 @@ class PosixFileSystem : public FileSystem { public: PosixFileSystem(); - const char* Name() const override { return "Posix File System"; } + static const char* kClassName() { return "PosixFileSystem"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const override { return kDefaultName(); } ~PosixFileSystem() override {} @@ -618,7 +620,8 @@ class PosixFileSystem : public FileSystem { return IOStatus::NotFound(); default: assert(err == EIO || err == ENOMEM); - return IOStatus::IOError("Unexpected error(" + ToString(err) + + return IOStatus::IOError("Unexpected error(" + + ROCKSDB_NAMESPACE::ToString(err) + ") accessing file `" + fname + "' "); } } @@ -819,11 +822,12 @@ class PosixFileSystem : public FileSystem { errno = ENOLCK; // Note that the thread ID printed is the same one as the one in // posix logger, but posix logger prints it hex format. - return IOError("lock hold by current process, acquire time " + - ToString(prev_info.acquire_time) + - " acquiring thread " + - ToString(prev_info.acquiring_thread), - fname, errno); + return IOError( + "lock hold by current process, acquire time " + + ROCKSDB_NAMESPACE::ToString(prev_info.acquire_time) + + " acquiring thread " + + ROCKSDB_NAMESPACE::ToString(prev_info.acquiring_thread), + fname, errno); } IOStatus result = IOStatus::OK(); diff --git a/env/fs_readonly.h b/env/fs_readonly.h index 89875106e..1bbe60784 100644 --- a/env/fs_readonly.h +++ b/env/fs_readonly.h @@ -26,6 +26,9 @@ class ReadOnlyFileSystem : public FileSystemWrapper { explicit ReadOnlyFileSystem(const std::shared_ptr& base) : FileSystemWrapper(base) {} + static const char* kClassName() { return "ReadOnlyFileSystem"; } + const char* Name() const override { return kClassName(); } + IOStatus NewWritableFile(const std::string& /*fname*/, const FileOptions& /*options*/, std::unique_ptr* /*result*/, diff --git a/env/fs_remap.h b/env/fs_remap.h index 4975822f6..1f6e061fd 100644 --- a/env/fs_remap.h +++ b/env/fs_remap.h @@ -39,6 +39,14 @@ class RemapFileSystem : public FileSystemWrapper { public: // Left abstract: // const char* Name() const override { ... } + static const char* kClassName() { return "RemapFileSystem"; } + bool IsInstanceOf(const std::string& id) const override { + if (id == kClassName()) { + return true; + } else { + return FileSystemWrapper::IsInstanceOf(id); + } + } Status RegisterDbPaths(const std::vector& paths) override; diff --git a/env/mock_env.cc b/env/mock_env.cc index e028f062e..c447a00d6 100644 --- a/env/mock_env.cc +++ b/env/mock_env.cc @@ -565,132 +565,39 @@ class TestMemLogger : public Logger { } size_t GetLogFileSize() const override { return log_size_; } }; +} // namespace -class MockFileSystem : public FileSystem { - public: - explicit MockFileSystem(const std::shared_ptr& clock, - bool supports_direct_io = true) - : system_clock_(clock), supports_direct_io_(supports_direct_io) { - clock_ = system_clock_.get(); +MockFileSystem::MockFileSystem(const std::shared_ptr& clock, + bool supports_direct_io) + : system_clock_(clock), supports_direct_io_(supports_direct_io) { + clock_ = system_clock_.get(); +} + +MockFileSystem::~MockFileSystem() { + for (auto i = file_map_.begin(); i != file_map_.end(); ++i) { + i->second->Unref(); } - - ~MockFileSystem() override { - for (auto i = file_map_.begin(); i != file_map_.end(); ++i) { - i->second->Unref(); - } +} +IOStatus MockFileSystem::GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* /*dbg*/) { + *output_path = NormalizeMockPath(db_path); + if (output_path->at(0) != '/') { + return IOStatus::NotSupported("GetAbsolutePath"); + } else { + return IOStatus::OK(); } +} - const char* Name() const override { return "Memory"; } - IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, - std::unique_ptr* r, - IODebugContext* dbg) override; - IOStatus NewRandomAccessFile(const std::string& f, - const FileOptions& file_opts, - std::unique_ptr* r, - IODebugContext* dbg) override; - - IOStatus NewRandomRWFile(const std::string& fname, - const FileOptions& file_opts, - std::unique_ptr* result, - IODebugContext* dbg) override; - IOStatus ReuseWritableFile(const std::string& fname, - const std::string& old_fname, - const FileOptions& file_opts, - std::unique_ptr* result, - IODebugContext* dbg) override; - IOStatus NewWritableFile(const std::string& fname, - const FileOptions& file_opts, - std::unique_ptr* result, - IODebugContext* dbg) override; - IOStatus ReopenWritableFile(const std::string& fname, - const FileOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override; - IOStatus NewDirectory(const std::string& /*name*/, const IOOptions& io_opts, - std::unique_ptr* result, - IODebugContext* dbg) override; - IOStatus FileExists(const std::string& fname, const IOOptions& /*io_opts*/, - IODebugContext* /*dbg*/) override; - IOStatus GetChildren(const std::string& dir, const IOOptions& options, - std::vector* result, - IODebugContext* dbg) override; - IOStatus DeleteFile(const std::string& fname, const IOOptions& options, - IODebugContext* dbg) override; - IOStatus Truncate(const std::string& fname, size_t size, - const IOOptions& options, IODebugContext* dbg) override; - IOStatus CreateDir(const std::string& dirname, const IOOptions& options, - IODebugContext* dbg) override; - IOStatus CreateDirIfMissing(const std::string& dirname, - const IOOptions& options, - IODebugContext* dbg) override; - IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, - IODebugContext* dbg) override; - - IOStatus GetFileSize(const std::string& fname, const IOOptions& options, - uint64_t* file_size, IODebugContext* dbg) override; - - IOStatus GetFileModificationTime(const std::string& fname, - const IOOptions& options, - uint64_t* file_mtime, - IODebugContext* dbg) override; - IOStatus RenameFile(const std::string& src, const std::string& target, - const IOOptions& options, IODebugContext* dbg) override; - IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/, - const IOOptions& /*options*/, - IODebugContext* /*dbg*/) override; - IOStatus LockFile(const std::string& fname, const IOOptions& options, - FileLock** lock, IODebugContext* dbg) override; - IOStatus UnlockFile(FileLock* lock, const IOOptions& options, - IODebugContext* dbg) override; - IOStatus GetTestDirectory(const IOOptions& options, std::string* path, - IODebugContext* dbg) override; - IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts, - std::shared_ptr* result, - IODebugContext* dbg) override; - // Get full directory name for this db. - IOStatus GetAbsolutePath(const std::string& db_path, - const IOOptions& /*options*/, - std::string* output_path, - IODebugContext* /*dbg*/) override { - *output_path = NormalizeMockPath(db_path); - if (output_path->at(0) != '/') { - return IOStatus::NotSupported("GetAbsolutePath"); - } else { - return IOStatus::OK(); - } - } - IOStatus IsDirectory(const std::string& /*path*/, - const IOOptions& /*options*/, bool* /*is_dir*/, - IODebugContext* /*dgb*/) override { - return IOStatus::NotSupported("IsDirectory"); +std::string MockFileSystem::NormalizeMockPath(const std::string& path) { + std::string p = NormalizePath(path); + if (p.back() == kFilePathSeparator && p.size() > 1) { + p.pop_back(); } + return p; +} - Status CorruptBuffer(const std::string& fname); - - private: - bool RenameFileInternal(const std::string& src, const std::string& dest); - void DeleteFileInternal(const std::string& fname); - bool GetChildrenInternal(const std::string& fname, - std::vector* results); - - std::string NormalizeMockPath(const std::string& path) { - std::string p = NormalizePath(path); - if (p.back() == kFilePathSeparator && p.size() > 1) { - p.pop_back(); - } - return p; - } - - private: - // Map from filenames to MemFile objects, representing a simple file system. - port::Mutex mutex_; - std::map file_map_; // Protected by mutex_. - std::shared_ptr system_clock_; - SystemClock* clock_; - bool supports_direct_io_; -}; - -} // Anonymous namespace // Partial implementation of the FileSystem interface. IOStatus MockFileSystem::NewSequentialFile( const std::string& fname, const FileOptions& file_opts, diff --git a/env/mock_env.h b/env/mock_env.h index 6c4b12f96..f2fc37dd8 100644 --- a/env/mock_env.h +++ b/env/mock_env.h @@ -14,11 +14,117 @@ #include #include "env/composite_env_wrapper.h" +#include "port/port.h" #include "rocksdb/env.h" #include "rocksdb/status.h" #include "rocksdb/system_clock.h" namespace ROCKSDB_NAMESPACE { +class MemFile; +class MockFileSystem : public FileSystem { + public: + explicit MockFileSystem(const std::shared_ptr& clock, + bool supports_direct_io = true); + ~MockFileSystem() override; + + static const char* kClassName() { return "MemoryFileSystem"; } + const char* Name() const override { return kClassName(); } + IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + IOStatus NewRandomAccessFile(const std::string& f, + const FileOptions& file_opts, + std::unique_ptr* r, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewWritableFile(const std::string& fname, + const FileOptions& file_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus ReopenWritableFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus NewDirectory(const std::string& /*name*/, const IOOptions& io_opts, + std::unique_ptr* result, + IODebugContext* dbg) override; + IOStatus FileExists(const std::string& fname, const IOOptions& /*io_opts*/, + IODebugContext* /*dbg*/) override; + IOStatus GetChildren(const std::string& dir, const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus Truncate(const std::string& fname, size_t size, + const IOOptions& options, IODebugContext* dbg) override; + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override; + IOStatus RenameFile(const std::string& src, const std::string& target, + const IOOptions& options, IODebugContext* dbg) override; + IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/, + const IOOptions& /*options*/, + IODebugContext* /*dbg*/) override; + IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) override; + IOStatus UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) override; + IOStatus GetTestDirectory(const IOOptions& options, std::string* path, + IODebugContext* dbg) override; + IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts, + std::shared_ptr* result, + IODebugContext* dbg) override; + // Get full directory name for this db. + IOStatus GetAbsolutePath(const std::string& db_path, + const IOOptions& /*options*/, + std::string* output_path, + IODebugContext* /*dbg*/) override; + IOStatus IsDirectory(const std::string& /*path*/, + const IOOptions& /*options*/, bool* /*is_dir*/, + IODebugContext* /*dgb*/) override { + return IOStatus::NotSupported("IsDirectory"); + } + + Status CorruptBuffer(const std::string& fname); + + private: + bool RenameFileInternal(const std::string& src, const std::string& dest); + void DeleteFileInternal(const std::string& fname); + bool GetChildrenInternal(const std::string& fname, + std::vector* results); + + std::string NormalizeMockPath(const std::string& path); + + private: + // Map from filenames to MemFile objects, representing a simple file system. + port::Mutex mutex_; + std::map file_map_; // Protected by mutex_. + std::shared_ptr system_clock_; + SystemClock* clock_; + bool supports_direct_io_; +}; + class MockEnv : public CompositeEnvWrapper { public: static MockEnv* Create(Env* base); diff --git a/file/prefetch_test.cc b/file/prefetch_test.cc index 739e0e37b..569326fe0 100644 --- a/file/prefetch_test.cc +++ b/file/prefetch_test.cc @@ -39,6 +39,9 @@ class MockFS : public FileSystemWrapper { bool support_prefetch) : FileSystemWrapper(wrapped), support_prefetch_(support_prefetch) {} + static const char* kClassName() { return "MockFS"; } + const char* Name() const override { return kClassName(); } + IOStatus NewRandomAccessFile(const std::string& fname, const FileOptions& opts, std::unique_ptr* result, diff --git a/include/rocksdb/env_encryption.h b/include/rocksdb/env_encryption.h index 239e6bb87..6fe011188 100644 --- a/include/rocksdb/env_encryption.h +++ b/include/rocksdb/env_encryption.h @@ -24,6 +24,9 @@ struct ConfigOptions; // read from disk. Env* NewEncryptedEnv(Env* base_env, const std::shared_ptr& provider); +std::shared_ptr NewEncryptedFS( + const std::shared_ptr& base_fs, + const std::shared_ptr& provider); // BlockAccessCipherStream is the base class for any cipher stream that // supports random access at block level (without requiring data from other @@ -440,6 +443,14 @@ class EncryptedFileSystem : public FileSystemWrapper { // otherwise virtual Status AddCipher(const std::string& descriptor, const char* cipher, size_t len, bool for_write) = 0; + static const char* kClassName() { return "EncryptedFileSystem"; } + bool IsInstanceOf(const std::string& name) const override { + if (name == kClassName()) { + return true; + } else { + return FileSystemWrapper::IsInstanceOf(name); + } + } }; } // namespace ROCKSDB_NAMESPACE diff --git a/include/rocksdb/file_system.h b/include/rocksdb/file_system.h index fb105bc6c..492a0dac7 100644 --- a/include/rocksdb/file_system.h +++ b/include/rocksdb/file_system.h @@ -28,6 +28,7 @@ #include #include +#include "rocksdb/customizable.h" #include "rocksdb/env.h" #include "rocksdb/io_status.h" #include "rocksdb/options.h" @@ -207,7 +208,8 @@ struct IODebugContext { // retryable. // NewCompositeEnv can be used to create an Env with a custom FileSystem for // DBOptions::env. -class FileSystem { + +class FileSystem : public Customizable { public: FileSystem(); @@ -216,9 +218,8 @@ class FileSystem { virtual ~FileSystem(); - virtual const char* Name() const = 0; - static const char* Type() { return "FileSystem"; } + static const char* kDefaultName() { return "DefaultFileSystem"; } // Loads the FileSystem specified by the input value into the result // The CreateFromString alternative should be used; this method may be @@ -1147,12 +1148,9 @@ class FSDirectory { class FileSystemWrapper : public FileSystem { public: // Initialize an EnvWrapper that delegates all calls to *t - explicit FileSystemWrapper(const std::shared_ptr& t) - : target_(t) {} + explicit FileSystemWrapper(const std::shared_ptr& t); ~FileSystemWrapper() override {} - const char* Name() const override { return target_->Name(); } - // Return the target to which this Env forwards all calls FileSystem* target() const { return target_.get(); } @@ -1343,7 +1341,13 @@ class FileSystemWrapper : public FileSystem { return target_->IsDirectory(path, options, is_dir, dbg); } - private: + const Customizable* Inner() const override { return target_.get(); } + Status PrepareOptions(const ConfigOptions& options) override; +#ifndef ROCKSDB_LITE + std::string SerializeOptions(const ConfigOptions& config_options, + const std::string& header) const override; +#endif // ROCKSDB_LITE + protected: std::shared_ptr target_; }; diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 4ca04beff..776393c8f 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -1299,6 +1299,17 @@ class MockCipher : public BlockCipher { Status Encrypt(char* /*data*/) override { return Status::NotSupported(); } Status Decrypt(char* data) override { return Encrypt(data); } }; +#endif // ROCKSDB_LITE + +class DummyFileSystem : public FileSystemWrapper { + public: + explicit DummyFileSystem(const std::shared_ptr& t) + : FileSystemWrapper(t) {} + static const char* kClassName() { return "DummyFileSystem"; } + const char* Name() const override { return kClassName(); } +}; + +#ifndef ROCKSDB_LITE #endif // ROCKSDB_LITE @@ -1406,6 +1417,14 @@ static int RegisterLocalObjects(ObjectLibrary& library, return guard->get(); }); + library.Register( + DummyFileSystem::kClassName(), + [](const std::string& /*uri*/, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new DummyFileSystem(nullptr)); + return guard->get(); + }); + library.Register( MockSstPartitionerFactory::kClassName(), [](const std::string& /*uri*/, @@ -1495,6 +1514,24 @@ TEST_F(LoadCustomizableTest, LoadTableFactoryTest) { } } +TEST_F(LoadCustomizableTest, LoadFileSystemTest) { + ColumnFamilyOptions cf_opts; + std::shared_ptr result; + ASSERT_NOK(FileSystem::CreateFromString( + config_options_, DummyFileSystem::kClassName(), &result)); + ASSERT_OK(FileSystem::CreateFromString(config_options_, + FileSystem::kDefaultName(), &result)); + ASSERT_NE(result, nullptr); + ASSERT_TRUE(result->IsInstanceOf(FileSystem::kDefaultName())); + if (RegisterTests("Test")) { + ASSERT_OK(FileSystem::CreateFromString( + config_options_, DummyFileSystem::kClassName(), &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), DummyFileSystem::kClassName()); + ASSERT_FALSE(result->IsInstanceOf(FileSystem::kDefaultName())); + } +} + TEST_F(LoadCustomizableTest, LoadSecondaryCacheTest) { std::shared_ptr result; ASSERT_NOK(SecondaryCache::CreateFromString( diff --git a/port/win/env_win.h b/port/win/env_win.h index e86515979..d64b82c92 100644 --- a/port/win/env_win.h +++ b/port/win/env_win.h @@ -110,7 +110,10 @@ class WinFileSystem : public FileSystem { static const std::shared_ptr& Default(); WinFileSystem(const std::shared_ptr& clock); ~WinFileSystem() {} - const char* Name() const { return "WinFS"; } + static const char* kClassName() { return "WinFS"; } + const char* Name() const override { return kClassName(); } + const char* NickName() const { return kDefaultName(); } + static size_t GetSectorSize(const std::string& fname); size_t GetPageSize() const { return page_size_; } size_t GetAllocationGranularity() const { return allocation_granularity_; } diff --git a/test_util/testutil.h b/test_util/testutil.h index b0424947a..709438293 100644 --- a/test_util/testutil.h +++ b/test_util/testutil.h @@ -555,6 +555,9 @@ class StringFS : public FileSystemWrapper { : FileSystemWrapper(t) {} ~StringFS() override {} + static const char* kClassName() { return "StringFS"; } + const char* Name() const override { return kClassName(); } + const std::string& GetContent(const std::string& f) { return files_[f]; } const IOStatus WriteToNewFile(const std::string& file_name, diff --git a/utilities/env_timed.cc b/utilities/env_timed.cc index 9fac82c5c..1eb723146 100644 --- a/utilities/env_timed.cc +++ b/utilities/env_timed.cc @@ -2,6 +2,7 @@ // This source code is licensed under both the GPLv2 (found in the // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). +#include "utilities/env_timed.h" #include "env/composite_env_wrapper.h" #include "monitoring/perf_context_imp.h" @@ -12,154 +13,157 @@ namespace ROCKSDB_NAMESPACE { #ifndef ROCKSDB_LITE -namespace { -class TimedFileSystem : public FileSystemWrapper { - public: - explicit TimedFileSystem(const std::shared_ptr& base) - : FileSystemWrapper(base) {} +TimedFileSystem::TimedFileSystem(const std::shared_ptr& base) + : FileSystemWrapper(base) {} +IOStatus TimedFileSystem::NewSequentialFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + PERF_TIMER_GUARD(env_new_sequential_file_nanos); + return FileSystemWrapper::NewSequentialFile(fname, options, result, dbg); +} - const char* Name() const override { return "TimedFS"; } - IOStatus NewSequentialFile(const std::string& fname, - const FileOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_new_sequential_file_nanos); - return FileSystemWrapper::NewSequentialFile(fname, options, result, dbg); - } +IOStatus TimedFileSystem::NewRandomAccessFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + PERF_TIMER_GUARD(env_new_random_access_file_nanos); + return FileSystemWrapper::NewRandomAccessFile(fname, options, result, dbg); +} - IOStatus NewRandomAccessFile(const std::string& fname, - const FileOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_new_random_access_file_nanos); - return FileSystemWrapper::NewRandomAccessFile(fname, options, result, dbg); - } +IOStatus TimedFileSystem::NewWritableFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + PERF_TIMER_GUARD(env_new_writable_file_nanos); + return FileSystemWrapper::NewWritableFile(fname, options, result, dbg); +} - IOStatus NewWritableFile(const std::string& fname, const FileOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_new_writable_file_nanos); - return FileSystemWrapper::NewWritableFile(fname, options, result, dbg); - } +IOStatus TimedFileSystem::ReuseWritableFile( + const std::string& fname, const std::string& old_fname, + const FileOptions& options, std::unique_ptr* result, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_reuse_writable_file_nanos); + return FileSystemWrapper::ReuseWritableFile(fname, old_fname, options, result, + dbg); +} - IOStatus ReuseWritableFile(const std::string& fname, - const std::string& old_fname, - const FileOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_reuse_writable_file_nanos); - return FileSystemWrapper::ReuseWritableFile(fname, old_fname, options, - result, dbg); - } +IOStatus TimedFileSystem::NewRandomRWFile( + const std::string& fname, const FileOptions& options, + std::unique_ptr* result, IODebugContext* dbg) { + PERF_TIMER_GUARD(env_new_random_rw_file_nanos); + return FileSystemWrapper::NewRandomRWFile(fname, options, result, dbg); +} - IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_new_random_rw_file_nanos); - return FileSystemWrapper::NewRandomRWFile(fname, options, result, dbg); - } +IOStatus TimedFileSystem::NewDirectory(const std::string& name, + const IOOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_new_directory_nanos); + return FileSystemWrapper::NewDirectory(name, options, result, dbg); +} - IOStatus NewDirectory(const std::string& name, const IOOptions& options, - std::unique_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_new_directory_nanos); - return FileSystemWrapper::NewDirectory(name, options, result, dbg); - } - - IOStatus FileExists(const std::string& fname, const IOOptions& options, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_file_exists_nanos); - return FileSystemWrapper::FileExists(fname, options, dbg); - } - - IOStatus GetChildren(const std::string& dir, const IOOptions& options, - std::vector* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_get_children_nanos); - return FileSystemWrapper::GetChildren(dir, options, result, dbg); - } - - IOStatus GetChildrenFileAttributes(const std::string& dir, +IOStatus TimedFileSystem::FileExists(const std::string& fname, const IOOptions& options, - std::vector* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_get_children_file_attributes_nanos); - return FileSystemWrapper::GetChildrenFileAttributes(dir, options, result, - dbg); - } + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_file_exists_nanos); + return FileSystemWrapper::FileExists(fname, options, dbg); +} - IOStatus DeleteFile(const std::string& fname, const IOOptions& options, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_delete_file_nanos); - return FileSystemWrapper::DeleteFile(fname, options, dbg); - } +IOStatus TimedFileSystem::GetChildren(const std::string& dir, + const IOOptions& options, + std::vector* result, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_get_children_nanos); + return FileSystemWrapper::GetChildren(dir, options, result, dbg); +} - IOStatus CreateDir(const std::string& dirname, const IOOptions& options, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_create_dir_nanos); - return FileSystemWrapper::CreateDir(dirname, options, dbg); - } +IOStatus TimedFileSystem::GetChildrenFileAttributes( + const std::string& dir, const IOOptions& options, + std::vector* result, IODebugContext* dbg) { + PERF_TIMER_GUARD(env_get_children_file_attributes_nanos); + return FileSystemWrapper::GetChildrenFileAttributes(dir, options, result, + dbg); +} - IOStatus CreateDirIfMissing(const std::string& dirname, - const IOOptions& options, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_create_dir_if_missing_nanos); - return FileSystemWrapper::CreateDirIfMissing(dirname, options, dbg); - } +IOStatus TimedFileSystem::DeleteFile(const std::string& fname, + const IOOptions& options, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_delete_file_nanos); + return FileSystemWrapper::DeleteFile(fname, options, dbg); +} - IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_delete_dir_nanos); - return FileSystemWrapper::DeleteDir(dirname, options, dbg); - } +IOStatus TimedFileSystem::CreateDir(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_create_dir_nanos); + return FileSystemWrapper::CreateDir(dirname, options, dbg); +} - IOStatus GetFileSize(const std::string& fname, const IOOptions& options, - uint64_t* file_size, IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_get_file_size_nanos); - return FileSystemWrapper::GetFileSize(fname, options, file_size, dbg); - } +IOStatus TimedFileSystem::CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_create_dir_if_missing_nanos); + return FileSystemWrapper::CreateDirIfMissing(dirname, options, dbg); +} - IOStatus GetFileModificationTime(const std::string& fname, +IOStatus TimedFileSystem::DeleteDir(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_delete_dir_nanos); + return FileSystemWrapper::DeleteDir(dirname, options, dbg); +} + +IOStatus TimedFileSystem::GetFileSize(const std::string& fname, + const IOOptions& options, + uint64_t* file_size, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_get_file_size_nanos); + return FileSystemWrapper::GetFileSize(fname, options, file_size, dbg); +} + +IOStatus TimedFileSystem::GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_get_file_modification_time_nanos); + return FileSystemWrapper::GetFileModificationTime(fname, options, file_mtime, + dbg); +} + +IOStatus TimedFileSystem::RenameFile(const std::string& src, + const std::string& dst, + const IOOptions& options, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_rename_file_nanos); + return FileSystemWrapper::RenameFile(src, dst, options, dbg); +} + +IOStatus TimedFileSystem::LinkFile(const std::string& src, + const std::string& dst, const IOOptions& options, - uint64_t* file_mtime, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_get_file_modification_time_nanos); - return FileSystemWrapper::GetFileModificationTime(fname, options, - file_mtime, dbg); - } + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_link_file_nanos); + return FileSystemWrapper::LinkFile(src, dst, options, dbg); +} - IOStatus RenameFile(const std::string& src, const std::string& dst, - const IOOptions& options, IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_rename_file_nanos); - return FileSystemWrapper::RenameFile(src, dst, options, dbg); - } +IOStatus TimedFileSystem::LockFile(const std::string& fname, + const IOOptions& options, FileLock** lock, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_lock_file_nanos); + return FileSystemWrapper::LockFile(fname, options, lock, dbg); +} - IOStatus LinkFile(const std::string& src, const std::string& dst, - const IOOptions& options, IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_link_file_nanos); - return FileSystemWrapper::LinkFile(src, dst, options, dbg); - } +IOStatus TimedFileSystem::UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_unlock_file_nanos); + return FileSystemWrapper::UnlockFile(lock, options, dbg); +} - IOStatus LockFile(const std::string& fname, const IOOptions& options, - FileLock** lock, IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_lock_file_nanos); - return FileSystemWrapper::LockFile(fname, options, lock, dbg); - } - - IOStatus UnlockFile(FileLock* lock, const IOOptions& options, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_unlock_file_nanos); - return FileSystemWrapper::UnlockFile(lock, options, dbg); - } - - IOStatus NewLogger(const std::string& fname, const IOOptions& options, - std::shared_ptr* result, - IODebugContext* dbg) override { - PERF_TIMER_GUARD(env_new_logger_nanos); - return FileSystemWrapper::NewLogger(fname, options, result, dbg); - } -}; -} // namespace +IOStatus TimedFileSystem::NewLogger(const std::string& fname, + const IOOptions& options, + std::shared_ptr* result, + IODebugContext* dbg) { + PERF_TIMER_GUARD(env_new_logger_nanos); + return FileSystemWrapper::NewLogger(fname, options, result, dbg); +} std::shared_ptr NewTimedFileSystem( const std::shared_ptr& base) { diff --git a/utilities/env_timed.h b/utilities/env_timed.h new file mode 100644 index 000000000..2d34fd590 --- /dev/null +++ b/utilities/env_timed.h @@ -0,0 +1,97 @@ +// Copyright (c) 2019-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// + +#pragma once +#include "rocksdb/file_system.h" +namespace ROCKSDB_NAMESPACE { +#ifndef ROCKSDB_LITE +class TimedFileSystem : public FileSystemWrapper { + public: + explicit TimedFileSystem(const std::shared_ptr& base); + + static const char* kClassName() { return "TimedFS"; } + const char* Name() const override { return kClassName(); } + + IOStatus NewSequentialFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomAccessFile(const std::string& fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewWritableFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus ReuseWritableFile(const std::string& fname, + const std::string& old_fname, + const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus NewDirectory(const std::string& name, const IOOptions& options, + std::unique_ptr* result, + IODebugContext* dbg) override; + + IOStatus FileExists(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetChildren(const std::string& dir, const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + + IOStatus GetChildrenFileAttributes(const std::string& dir, + const IOOptions& options, + std::vector* result, + IODebugContext* dbg) override; + + IOStatus DeleteFile(const std::string& fname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus CreateDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus CreateDirIfMissing(const std::string& dirname, + const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus DeleteDir(const std::string& dirname, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus GetFileSize(const std::string& fname, const IOOptions& options, + uint64_t* file_size, IODebugContext* dbg) override; + + IOStatus GetFileModificationTime(const std::string& fname, + const IOOptions& options, + uint64_t* file_mtime, + IODebugContext* dbg) override; + + IOStatus RenameFile(const std::string& src, const std::string& dst, + const IOOptions& options, IODebugContext* dbg) override; + + IOStatus LinkFile(const std::string& src, const std::string& dst, + const IOOptions& options, IODebugContext* dbg) override; + + IOStatus LockFile(const std::string& fname, const IOOptions& options, + FileLock** lock, IODebugContext* dbg) override; + + IOStatus UnlockFile(FileLock* lock, const IOOptions& options, + IODebugContext* dbg) override; + + IOStatus NewLogger(const std::string& fname, const IOOptions& options, + std::shared_ptr* result, + IODebugContext* dbg) override; +}; + +#endif // ROCKSDB_LITE +} // namespace ROCKSDB_NAMESPACE diff --git a/utilities/fault_injection_fs.h b/utilities/fault_injection_fs.h index 896597849..868b9809c 100644 --- a/utilities/fault_injection_fs.h +++ b/utilities/fault_injection_fs.h @@ -200,7 +200,8 @@ class FaultInjectionTestFS : public FileSystemWrapper { fail_get_file_unique_id_(false) {} virtual ~FaultInjectionTestFS() { error_.PermitUncheckedError(); } - const char* Name() const override { return "FaultInjectionTestFS"; } + static const char* kClassName() { return "FaultInjectionTestFS"; } + const char* Name() const override { return kClassName(); } IOStatus NewDirectory(const std::string& name, const IOOptions& options, std::unique_ptr* result,