From 42ba60b3bac443fd1c8de40c660123b6000b5820 Mon Sep 17 00:00:00 2001 From: Mark Rambacher Date: Fri, 16 Jul 2021 07:57:47 -0700 Subject: [PATCH] Make EncryptionProvider and BlockCipher into Customizable objects (#8354) Summary: Made the EncryptionProvider and BlockCipher classes inherit from Customizable. Added/fixed the CreateFromString method to these classes to create instances from builtin or registered classes. Added tests to verify that instances can be registered and retrieved as appropriate. Added the ability to configure the builtin (CTR, ROT13) classes from configurable properties. Added the appropriate tests. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8354 Reviewed By: zhichao-cao Differential Revision: D29558949 Pulled By: mrambacher fbshipit-source-id: c20286b32d179777e060f51a58943e9b0cf81d04 --- db/db_test_util.cc | 10 +- env/env_basic_test.cc | 132 ++++++++++++++-------- env/env_encryption.cc | 185 +++++++++++++++++++------------ env/env_encryption_ctr.h | 33 +----- env/env_test.cc | 71 ++++++++++++ include/rocksdb/env_encryption.h | 23 ++-- options/configurable.cc | 42 +++---- options/customizable_test.cc | 131 +++++++++++++++++++--- 8 files changed, 428 insertions(+), 199 deletions(-) diff --git a/db/db_test_util.cc b/db/db_test_util.cc index ef10ec479..1f425d582 100644 --- a/db/db_test_util.cc +++ b/db/db_test_util.cc @@ -68,9 +68,13 @@ DBTestBase::DBTestBase(const std::string path, bool env_do_fsync) #ifndef ROCKSDB_LITE if (getenv("ENCRYPTED_ENV")) { std::shared_ptr provider; - Status s = EncryptionProvider::CreateFromString( - config_options, std::string("test://") + getenv("ENCRYPTED_ENV"), - &provider); + std::string provider_id = getenv("ENCRYPTED_ENV"); + if (provider_id.find("=") == std::string::npos && + !EndsWith(provider_id, "://test")) { + provider_id = provider_id + "://test"; + } + EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id, + &provider)); encrypted_env_ = NewEncryptedEnv(mem_env_ ? mem_env_ : base_env, provider); } #endif // !ROCKSDB_LITE diff --git a/env/env_basic_test.cc b/env/env_basic_test.cc index e8e3df5f6..3d7423944 100644 --- a/env/env_basic_test.cc +++ b/env/env_basic_test.cc @@ -17,15 +17,81 @@ #include "test_util/testharness.h" namespace ROCKSDB_NAMESPACE { +typedef Env* CreateEnvFunc(); +namespace { +// These functions are used to create the various environments under which this +// test can execute. These functions are used to allow the test cases to be +// created without the Env being initialized, thereby eliminating a potential +// static initialization fiasco/race condition when attempting to get a +// custom/configured env prior to main being invoked. -class EnvBasicTestWithParam : public testing::Test, - public ::testing::WithParamInterface { +static Env* GetDefaultEnv() { return Env::Default(); } + +static Env* GetMockEnv() { + static std::unique_ptr mock_env(new MockEnv(Env::Default())); + return mock_env.get(); +} +#ifndef ROCKSDB_LITE +static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) { + ConfigOptions config_opts; + config_opts.invoke_prepare_options = false; + + std::shared_ptr provider; + EXPECT_OK(EncryptionProvider::CreateFromString(config_opts, provider_id, + &provider)); + return NewEncryptedEnv(base, provider); +} + +static Env* GetCtrEncryptedEnv() { + static std::unique_ptr ctr_encrypt_env( + NewTestEncryptedEnv(Env::Default(), "CTR://test")); + return ctr_encrypt_env.get(); +} + +static Env* GetMemoryEnv() { + static std::unique_ptr mem_env(NewMemEnv(Env::Default())); + return mem_env.get(); +} + +static Env* GetTestEnv() { + static std::shared_ptr env_guard; + static Env* custom_env = nullptr; + if (custom_env == nullptr) { + const char* uri = getenv("TEST_ENV_URI"); + if (uri != nullptr) { + EXPECT_OK(Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, + &env_guard)); + } + } + EXPECT_NE(custom_env, nullptr); + return custom_env; +} + +static Env* GetTestFS() { + static std::shared_ptr fs_env_guard; + static Env* fs_env = nullptr; + if (fs_env == nullptr) { + const char* uri = getenv("TEST_FS_URI"); + if (uri != nullptr) { + EXPECT_OK( + Env::CreateFromUri(ConfigOptions(), uri, "", &fs_env, &fs_env_guard)); + } + } + EXPECT_NE(fs_env, nullptr); + return fs_env; +} +#endif // ROCKSDB_LITE + +} // namespace +class EnvBasicTestWithParam + : public testing::Test, + public ::testing::WithParamInterface { public: Env* env_; const EnvOptions soptions_; std::string test_dir_; - EnvBasicTestWithParam() : env_(GetParam()) { + EnvBasicTestWithParam() : env_(GetParam()()) { test_dir_ = test::PerThreadDBPath(env_, "env_basic_test"); } @@ -37,35 +103,22 @@ class EnvBasicTestWithParam : public testing::Test, class EnvMoreTestWithParam : public EnvBasicTestWithParam {}; INSTANTIATE_TEST_CASE_P(EnvDefault, EnvBasicTestWithParam, - ::testing::Values(Env::Default())); + ::testing::Values(&GetDefaultEnv)); INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam, - ::testing::Values(Env::Default())); + ::testing::Values(&GetDefaultEnv)); -static std::unique_ptr mock_env(new MockEnv(Env::Default())); INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam, - ::testing::Values(mock_env.get())); + ::testing::Values(&GetMockEnv)); #ifndef ROCKSDB_LITE -static Env* NewTestEncryptedEnv(Env* base, const std::string& provider_id) { - std::shared_ptr provider; - EXPECT_OK(EncryptionProvider::CreateFromString(ConfigOptions(), provider_id, - &provider)); - return NewEncryptedEnv(base, provider); -} - // next statements run env test against default encryption code. -static std::unique_ptr ctr_encrypt_env(NewTestEncryptedEnv(Env::Default(), - "test://CTR")); INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam, - ::testing::Values(ctr_encrypt_env.get())); + ::testing::Values(&GetCtrEncryptedEnv)); INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam, - ::testing::Values(ctr_encrypt_env.get())); -#endif // ROCKSDB_LITE + ::testing::Values(&GetCtrEncryptedEnv)); -#ifndef ROCKSDB_LITE -static std::unique_ptr mem_env(NewMemEnv(Env::Default())); INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam, - ::testing::Values(mem_env.get())); + ::testing::Values(&GetMemoryEnv)); namespace { @@ -74,31 +127,15 @@ namespace { // // The purpose of returning an empty vector (instead of nullptr) is that gtest // ValuesIn() will skip running tests when given an empty collection. -std::vector GetCustomEnvs() { - static bool init = false; - static std::vector res; - if (!init) { - init = true; - const char* uri = getenv("TEST_ENV_URI"); - if (uri != nullptr) { - static std::shared_ptr env_guard; - static Env* custom_env; - Status s = - Env::CreateFromUri(ConfigOptions(), uri, "", &custom_env, &env_guard); - if (s.ok()) { - res.emplace_back(custom_env); - } - } - uri = getenv("TEST_FS_URI"); - if (uri != nullptr) { - static std::shared_ptr fs_env_guard; - static Env* fs_env; - Status s = - Env::CreateFromUri(ConfigOptions(), "", uri, &fs_env, &fs_env_guard); - if (s.ok()) { - res.emplace_back(fs_env); - } - } +std::vector GetCustomEnvs() { + std::vector res; + const char* uri = getenv("TEST_ENV_URI"); + if (uri != nullptr) { + res.push_back(&GetTestEnv); + } + uri = getenv("TEST_FS_URI"); + if (uri != nullptr) { + res.push_back(&GetTestFS); } return res; } @@ -110,7 +147,6 @@ INSTANTIATE_TEST_CASE_P(CustomEnv, EnvBasicTestWithParam, INSTANTIATE_TEST_CASE_P(CustomEnv, EnvMoreTestWithParam, ::testing::ValuesIn(GetCustomEnvs())); - #endif // ROCKSDB_LITE TEST_P(EnvBasicTestWithParam, Basics) { diff --git a/env/env_encryption.cc b/env/env_encryption.cc index a5670ad78..22e2e5190 100644 --- a/env/env_encryption.cc +++ b/env/env_encryption.cc @@ -18,62 +18,16 @@ #include "rocksdb/convenience.h" #include "rocksdb/io_status.h" #include "rocksdb/system_clock.h" +#include "rocksdb/utilities/customizable_util.h" +#include "rocksdb/utilities/options_type.h" #include "util/aligned_buffer.h" #include "util/coding.h" #include "util/random.h" #include "util/string_util.h" #endif - namespace ROCKSDB_NAMESPACE { - #ifndef ROCKSDB_LITE -static constexpr char kROT13CipherName[] = "ROT13"; -static constexpr char kCTRProviderName[] = "CTR"; - -Status BlockCipher::CreateFromString(const ConfigOptions& /*config_options*/, - const std::string& value, - std::shared_ptr* result) { - std::string id = value; - size_t colon = value.find(':'); - if (colon != std::string::npos) { - id = value.substr(0, colon); - } - if (id == kROT13CipherName) { - if (colon != std::string::npos) { - size_t block_size = ParseSizeT(value.substr(colon + 1)); - result->reset(new ROT13BlockCipher(block_size)); - } else { - result->reset(new ROT13BlockCipher(32)); - } - return Status::OK(); - } else { - return Status::NotSupported("Could not find cipher ", value); - } -} - -Status EncryptionProvider::CreateFromString( - const ConfigOptions& /*config_options*/, const std::string& value, - std::shared_ptr* result) { - std::string id = value; - bool is_test = StartsWith(value, "test://"); - Status status = Status::OK(); - if (is_test) { - id = value.substr(strlen("test://")); - } - if (id == kCTRProviderName) { - result->reset(new CTREncryptionProvider()); - } else if (is_test) { - result->reset(new CTREncryptionProvider()); - } else { - return Status::NotSupported("Could not find provider ", value); - } - if (status.ok() && is_test) { - status = result->get()->TEST_Initialize(); - } - return status; -} - std::shared_ptr EncryptionProvider::NewCTRProvider( const std::shared_ptr& cipher) { return std::make_shared(cipher); @@ -1061,20 +1015,53 @@ Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char *data, size_t } } -const char* ROT13BlockCipher::Name() const { return kROT13CipherName; } +namespace { +static std::unordered_map + rot13_block_cipher_type_info = { + {"block_size", + {0 /* No offset, whole struct*/, OptionType::kInt, + OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, +}; +// Implements a BlockCipher using ROT13. +// +// Note: This is a sample implementation of BlockCipher, +// it is NOT considered safe and should NOT be used in production. +class ROT13BlockCipher : public BlockCipher { + private: + size_t blockSize_; -// Encrypt a block of data. -// Length of data is equal to BlockSize(). -Status ROT13BlockCipher::Encrypt(char* data) { - for (size_t i = 0; i < blockSize_; ++i) { - data[i] += 13; + public: + explicit ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) { + RegisterOptions("ROT13BlockCipherOptions", &blockSize_, + &rot13_block_cipher_type_info); } - return Status::OK(); -} -// Decrypt a block of data. -// Length of data is equal to BlockSize(). -Status ROT13BlockCipher::Decrypt(char* data) { return Encrypt(data); } + static const char* kClassName() { return "ROT13"; } + const char* Name() const override { return kClassName(); } + // BlockSize returns the size of each block supported by this cipher stream. + size_t BlockSize() override { return blockSize_; } + + // Encrypt a block of data. + // Length of data is equal to BlockSize(). + Status Encrypt(char* data) override { + for (size_t i = 0; i < blockSize_; ++i) { + data[i] += 13; + } + return Status::OK(); + } + + // Decrypt a block of data. + // Length of data is equal to BlockSize(). + Status Decrypt(char* data) override { return Encrypt(data); } +}; +static const std::unordered_map + ctr_encryption_provider_type_info = { + {"cipher", + OptionTypeInfo::AsCustomSharedPtr( + 0 /* No offset, whole struct*/, OptionVerificationType::kByName, + OptionTypeFlags::kNone)}, +}; +} // anonymous namespace // Allocate scratch space which is passed to EncryptBlock/DecryptBlock. void CTRCipherStream::AllocateScratch(std::string& scratch) { @@ -1112,7 +1099,11 @@ Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data, return EncryptBlock(blockIndex, data, scratch); } -const char* CTREncryptionProvider::Name() const { return kCTRProviderName; } +CTREncryptionProvider::CTREncryptionProvider( + const std::shared_ptr& c) + : cipher_(c) { + RegisterOptions("Cipher", &cipher_, &ctr_encryption_provider_type_info); +} // GetPrefixLength returns the length of the prefix that is added to every file // and used for storing encryption options. @@ -1122,20 +1113,12 @@ size_t CTREncryptionProvider::GetPrefixLength() const { return defaultPrefixLength; } -Status CTREncryptionProvider::TEST_Initialize() { - if (!cipher_) { - return BlockCipher::CreateFromString( - ConfigOptions(), std::string(kROT13CipherName) + ":32", &cipher_); - } - return Status::OK(); -} - Status CTREncryptionProvider::AddCipher(const std::string& /*descriptor*/, const char* cipher, size_t len, bool /*for_write*/) { if (cipher_) { return Status::NotSupported("Cannot add keys to CTREncryptionProvider"); - } else if (strcmp(kROT13CipherName, cipher) == 0) { + } else if (strcmp(ROT13BlockCipher::kClassName(), cipher) == 0) { cipher_.reset(new ROT13BlockCipher(len)); return Status::OK(); } else { @@ -1252,6 +1235,70 @@ Status CTREncryptionProvider::CreateCipherStreamFromPrefix( return Status::OK(); } +namespace { +static void RegisterEncryptionBuiltins() { + static std::once_flag once; + std::call_once(once, [&]() { + auto lib = ObjectRegistry::Default()->AddLibrary("encryption"); + std::string ctr = + std::string(CTREncryptionProvider::kClassName()) + "?(://test)"; + lib->Register( + std::string(CTREncryptionProvider::kClassName()) + "(://test)?", + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + if (EndsWith(uri, "://test")) { + std::shared_ptr cipher = + std::make_shared(32); + guard->reset(new CTREncryptionProvider(cipher)); + } else { + guard->reset(new CTREncryptionProvider()); + } + return guard->get(); + }); + + lib->Register( + "1://test", [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /*errmsg*/) { + std::shared_ptr cipher = + std::make_shared(32); + guard->reset(new CTREncryptionProvider(cipher)); + return guard->get(); + }); + + lib->Register( + std::string(ROT13BlockCipher::kClassName()) + "(:.*)?", + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + size_t colon = uri.find(':'); + if (colon != std::string::npos) { + size_t block_size = ParseSizeT(uri.substr(colon + 1)); + guard->reset(new ROT13BlockCipher(block_size)); + } else { + guard->reset(new ROT13BlockCipher(32)); + } + + return guard->get(); + }); + }); +} +} // namespace + +Status BlockCipher::CreateFromString(const ConfigOptions& config_options, + const std::string& value, + std::shared_ptr* result) { + RegisterEncryptionBuiltins(); + return LoadSharedObject(config_options, value, nullptr, result); +} + +Status EncryptionProvider::CreateFromString( + const ConfigOptions& config_options, const std::string& value, + std::shared_ptr* result) { + RegisterEncryptionBuiltins(); + return LoadSharedObject(config_options, value, nullptr, + result); +} + #endif // ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE diff --git a/env/env_encryption_ctr.h b/env/env_encryption_ctr.h index b22d7e45f..f034708c2 100644 --- a/env/env_encryption_ctr.h +++ b/env/env_encryption_ctr.h @@ -10,31 +10,6 @@ #include "rocksdb/env_encryption.h" namespace ROCKSDB_NAMESPACE { - -// Implements a BlockCipher using ROT13. -// -// Note: This is a sample implementation of BlockCipher, -// it is NOT considered safe and should NOT be used in production. -class ROT13BlockCipher : public BlockCipher { - private: - size_t blockSize_; - - public: - ROT13BlockCipher(size_t blockSize) : blockSize_(blockSize) {} - virtual ~ROT13BlockCipher(){}; - const char* Name() const override; - // BlockSize returns the size of each block supported by this cipher stream. - size_t BlockSize() override { return blockSize_; } - - // Encrypt a block of data. - // Length of data is equal to BlockSize(). - Status Encrypt(char* data) override; - - // Decrypt a block of data. - // Length of data is equal to BlockSize(). - Status Decrypt(char* data) override; -}; - // CTRCipherStream implements BlockAccessCipherStream using an // Counter operations mode. // See https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation @@ -86,11 +61,11 @@ class CTREncryptionProvider : public EncryptionProvider { public: explicit CTREncryptionProvider( - const std::shared_ptr& c = nullptr) - : cipher_(c){}; + const std::shared_ptr& c = nullptr); virtual ~CTREncryptionProvider() {} - const char* Name() const override; + static const char* kClassName() { return "CTR"; } + const char* Name() const override { return kClassName(); } // GetPrefixLength returns the length of the prefix that is added to every // file @@ -112,9 +87,7 @@ class CTREncryptionProvider : public EncryptionProvider { Status AddCipher(const std::string& descriptor, const char* /*cipher*/, size_t /*len*/, bool /*for_write*/) override; - protected: - Status TEST_Initialize() override; // PopulateSecretPrefixPart initializes the data into a new prefix block // that will be encrypted. This function will store the data in plain text. diff --git a/env/env_test.cc b/env/env_test.cc index c72390473..ac128b6fc 100644 --- a/env/env_test.cc +++ b/env/env_test.cc @@ -36,10 +36,13 @@ #endif #include "env/env_chroot.h" +#include "env/env_encryption_ctr.h" #include "logging/log_buffer.h" #include "port/malloc.h" #include "port/port.h" +#include "rocksdb/convenience.h" #include "rocksdb/env.h" +#include "rocksdb/env_encryption.h" #include "rocksdb/system_clock.h" #include "test_util/sync_point.h" #include "test_util/testharness.h" @@ -2387,6 +2390,74 @@ TEST_F(EnvTest, EnvWriteVerificationTest) { ASSERT_OK(s); } +#ifndef ROCKSDB_LITE +class EncryptionProviderTest : public testing::Test { + public: +}; + +TEST_F(EncryptionProviderTest, LoadCTRProvider) { + ConfigOptions config_options; + config_options.invoke_prepare_options = false; + std::string CTR = CTREncryptionProvider::kClassName(); + std::shared_ptr provider; + // Test a provider with no cipher + ASSERT_OK( + EncryptionProvider::CreateFromString(config_options, CTR, &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + ASSERT_NOK(provider->PrepareOptions(config_options)); + ASSERT_NOK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + auto cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_EQ(cipher->get(), nullptr); + provider.reset(); + + ASSERT_OK(EncryptionProvider::CreateFromString(config_options, + CTR + "://test", &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + ASSERT_OK(provider->PrepareOptions(config_options)); + ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_NE(cipher->get(), nullptr); + ASSERT_STREQ(cipher->get()->Name(), "ROT13"); + provider.reset(); + + ASSERT_OK(EncryptionProvider::CreateFromString(config_options, "1://test", + &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + ASSERT_OK(provider->PrepareOptions(config_options)); + ASSERT_OK(provider->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_NE(cipher->get(), nullptr); + ASSERT_STREQ(cipher->get()->Name(), "ROT13"); + provider.reset(); + + ASSERT_OK(EncryptionProvider::CreateFromString( + config_options, "id=" + CTR + "; cipher=ROT13", &provider)); + ASSERT_NE(provider, nullptr); + ASSERT_EQ(provider->Name(), CTR); + cipher = provider->GetOptions>("Cipher"); + ASSERT_NE(cipher, nullptr); + ASSERT_NE(cipher->get(), nullptr); + ASSERT_STREQ(cipher->get()->Name(), "ROT13"); + provider.reset(); +} + +TEST_F(EncryptionProviderTest, LoadROT13Cipher) { + ConfigOptions config_options; + std::shared_ptr cipher; + // Test a provider with no cipher + ASSERT_OK(BlockCipher::CreateFromString(config_options, "ROT13", &cipher)); + ASSERT_NE(cipher, nullptr); + ASSERT_STREQ(cipher->Name(), "ROT13"); +} + +#endif // ROCKSDB_LITE + } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/include/rocksdb/env_encryption.h b/include/rocksdb/env_encryption.h index 7a76ec867..e243e3eb7 100644 --- a/include/rocksdb/env_encryption.h +++ b/include/rocksdb/env_encryption.h @@ -9,6 +9,7 @@ #include +#include "rocksdb/customizable.h" #include "rocksdb/env.h" #include "rocksdb/file_system.h" #include "rocksdb/rocksdb_namespace.h" @@ -58,7 +59,7 @@ class BlockAccessCipherStream { }; // BlockCipher -class BlockCipher { +class BlockCipher : public Customizable { public: virtual ~BlockCipher(){}; @@ -80,12 +81,12 @@ class BlockCipher { const std::string& value, std::shared_ptr* result); + static const char* Type() { return "BlockCipher"; } // Short-cut method to create a ROT13 BlockCipher. // This cipher is only suitable for test purposes and should not be used in // production!!! static std::shared_ptr NewROT13Cipher(size_t block_size); - virtual const char* Name() const = 0; // BlockSize returns the size of each block supported by this cipher stream. virtual size_t BlockSize() = 0; @@ -101,7 +102,7 @@ class BlockCipher { // The encryption provider is used to create a cipher stream for a specific // file. The returned cipher stream will be used for actual // encryption/decryption actions. -class EncryptionProvider { +class EncryptionProvider : public Customizable { public: virtual ~EncryptionProvider(){}; @@ -109,14 +110,14 @@ class EncryptionProvider { // The value describes the type of provider (and potentially optional // configuration parameters) used to create this provider. // For example, if the value is "CTR", a CTREncryptionProvider will be - // created. If the value is preceded by "test://" (e.g test://CTR"), the - // TEST_Initialize method will be invoked prior to returning the provider. + // created. If the value is ends with "://test" (e.g CTR://test"), the + // provider will be initialized in "TEST" mode prior to being returned. // // @param config_options Options to control how this provider is created // and initialized. // @param value The value might be: // - CTR Create a CTR provider - // - test://CTR Create a CTR provider and initialize it for tests. + // - CTR://test Create a CTR provider and initialize it for tests. // @param result The new provider object // @return OK if the provider was successfully created // @return NotFound if an invalid name was specified in the value @@ -125,13 +126,12 @@ class EncryptionProvider { const std::string& value, std::shared_ptr* result); + static const char* Type() { return "EncryptionProvider"; } + // Short-cut method to create a CTR-provider static std::shared_ptr NewCTRProvider( const std::shared_ptr& cipher); - // Returns the name of this EncryptionProvider - virtual const char* Name() const = 0; - // GetPrefixLength returns the length of the prefix that is added to every // file and used for storing encryption options. For optimal performance, the // prefix length should be a multiple of the page size. @@ -165,11 +165,6 @@ class EncryptionProvider { // or not a file is encrypted by this provider. The maker will also be part // of any encryption prefix for this provider. virtual std::string GetMarker() const { return ""; } - - protected: - // Optional method to initialize an EncryptionProvider in the TEST - // environment. - virtual Status TEST_Initialize() { return Status::OK(); } }; class EncryptedSequentialFile : public FSSequentialFile { diff --git a/options/configurable.cc b/options/configurable.cc index 1a22f05c6..44ad5d865 100644 --- a/options/configurable.cc +++ b/options/configurable.cc @@ -40,34 +40,36 @@ void Configurable::RegisterOptions( //************************************************************************* Status Configurable::PrepareOptions(const ConfigOptions& opts) { + // We ignore the invoke_prepare_options here intentionally, + // as if you are here, you must have called PrepareOptions explicitly. Status status = Status::OK(); - if (opts.invoke_prepare_options) { #ifndef ROCKSDB_LITE - for (auto opt_iter : options_) { - for (auto map_iter : *(opt_iter.type_map)) { - auto& opt_info = map_iter.second; - if (!opt_info.IsDeprecated() && !opt_info.IsAlias() && - opt_info.IsConfigurable()) { - if (!opt_info.IsEnabled(OptionTypeFlags::kDontPrepare)) { - Configurable* config = - opt_info.AsRawPointer(opt_iter.opt_ptr); - if (config != nullptr) { - status = config->PrepareOptions(opts); - if (!status.ok()) { - return status; - } - } else if (!opt_info.CanBeNull()) { - status = Status::NotFound("Missing configurable object", - map_iter.first); + for (auto opt_iter : options_) { + for (auto map_iter : *(opt_iter.type_map)) { + auto& opt_info = map_iter.second; + if (!opt_info.IsDeprecated() && !opt_info.IsAlias() && + opt_info.IsConfigurable()) { + if (!opt_info.IsEnabled(OptionTypeFlags::kDontPrepare)) { + Configurable* config = + opt_info.AsRawPointer(opt_iter.opt_ptr); + if (config != nullptr) { + status = config->PrepareOptions(opts); + if (!status.ok()) { + return status; } + } else if (!opt_info.CanBeNull()) { + status = + Status::NotFound("Missing configurable object", map_iter.first); } } } } + } +#else + (void)opts; #endif // ROCKSDB_LITE - if (status.ok()) { - is_prepared_ = true; - } + if (status.ok()) { + is_prepared_ = true; } return status; } diff --git a/options/customizable_test.cc b/options/customizable_test.cc index 772f6ea36..68ed84f43 100644 --- a/options/customizable_test.cc +++ b/options/customizable_test.cc @@ -18,6 +18,7 @@ #include "options/options_helper.h" #include "options/options_parser.h" #include "rocksdb/convenience.h" +#include "rocksdb/env_encryption.h" #include "rocksdb/flush_block_policy.h" #include "rocksdb/secondary_cache.h" #include "rocksdb/utilities/customizable_util.h" @@ -27,6 +28,7 @@ #include "table/mock_table.h" #include "test_util/testharness.h" #include "test_util/testutil.h" +#include "util/string_util.h" #ifndef GFLAGS bool FLAGS_enable_print = false; @@ -231,15 +233,18 @@ static std::unordered_map simple_option_info = { {"bool", {offsetof(struct SimpleOptions, b), OptionType::kBoolean, OptionVerificationType::kNormal, OptionTypeFlags::kNone}}, - {"unique", OptionTypeInfo::AsCustomUniquePtr( - offsetof(struct SimpleOptions, cu), - OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, - {"shared", OptionTypeInfo::AsCustomSharedPtr( - offsetof(struct SimpleOptions, cs), - OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, - {"pointer", OptionTypeInfo::AsCustomRawPtr( - offsetof(struct SimpleOptions, cp), - OptionVerificationType::kNormal, OptionTypeFlags::kNone)}, + {"unique", + OptionTypeInfo::AsCustomUniquePtr( + offsetof(struct SimpleOptions, cu), OptionVerificationType::kNormal, + OptionTypeFlags::kAllowNull)}, + {"shared", + OptionTypeInfo::AsCustomSharedPtr( + offsetof(struct SimpleOptions, cs), OptionVerificationType::kNormal, + OptionTypeFlags::kAllowNull)}, + {"pointer", + OptionTypeInfo::AsCustomRawPtr( + offsetof(struct SimpleOptions, cp), OptionVerificationType::kNormal, + OptionTypeFlags::kAllowNull)}, #endif // ROCKSDB_LITE }; @@ -554,11 +559,6 @@ TEST_F(CustomizableTest, PrepareOptionsTest) { ASSERT_FALSE(simple->cp->IsPrepared()); ASSERT_OK(base->PrepareOptions(config_options_)); - ASSERT_FALSE(base->IsPrepared()); - ASSERT_FALSE(simple->cu->IsPrepared()); - ASSERT_FALSE(simple->cs->IsPrepared()); - ASSERT_FALSE(simple->cp->IsPrepared()); - ASSERT_OK(base->PrepareOptions(prepared)); ASSERT_TRUE(base->IsPrepared()); ASSERT_TRUE(simple->cu->IsPrepared()); ASSERT_TRUE(simple->cs->IsPrepared()); @@ -917,6 +917,48 @@ static int RegisterTestObjects(ObjectLibrary& library, return static_cast(library.GetFactoryCount(&num_types)); } +class MockEncryptionProvider : public EncryptionProvider { + public: + explicit MockEncryptionProvider(const std::string& id) : id_(id) {} + const char* Name() const override { return "Mock"; } + size_t GetPrefixLength() const override { return 0; } + Status CreateNewPrefix(const std::string& /*fname*/, char* /*prefix*/, + size_t /*prefixLength*/) const override { + return Status::NotSupported(); + } + + Status AddCipher(const std::string& /*descriptor*/, const char* /*cipher*/, + size_t /*len*/, bool /*for_write*/) override { + return Status::NotSupported(); + } + + Status CreateCipherStream( + const std::string& /*fname*/, const EnvOptions& /*options*/, + Slice& /*prefix*/, + std::unique_ptr* /*result*/) override { + return Status::NotSupported(); + } + Status ValidateOptions(const DBOptions& db_opts, + const ColumnFamilyOptions& cf_opts) const override { + if (EndsWith(id_, "://test")) { + return EncryptionProvider::ValidateOptions(db_opts, cf_opts); + } else { + return Status::InvalidArgument("MockProvider not initialized"); + } + } + + private: + std::string id_; +}; + +class MockCipher : public BlockCipher { + public: + const char* Name() const override { return "Mock"; } + size_t BlockSize() override { return 0; } + Status Encrypt(char* /*data*/) override { return Status::NotSupported(); } + Status Decrypt(char* data) override { return Encrypt(data); } +}; + class TestFlushBlockPolicyFactory : public FlushBlockPolicyFactory { public: TestFlushBlockPolicyFactory() {} @@ -935,6 +977,19 @@ static int RegisterLocalObjects(ObjectLibrary& library, const std::string& /*arg*/) { size_t num_types; // Load any locally defined objects here + library.Register( + "Mock(://test)?", + [](const std::string& uri, std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockEncryptionProvider(uri)); + return guard->get(); + }); + library.Register("Mock", [](const std::string& /*uri*/, + std::unique_ptr* guard, + std::string* /* errmsg */) { + guard->reset(new MockCipher()); + return guard->get(); + }); library.Register( TestFlushBlockPolicyFactory::kClassName(), [](const std::string& /*uri*/, @@ -957,7 +1012,10 @@ static int RegisterLocalObjects(ObjectLibrary& library, class LoadCustomizableTest : public testing::Test { public: - LoadCustomizableTest() { config_options_.ignore_unsupported_options = false; } + LoadCustomizableTest() { + config_options_.ignore_unsupported_options = false; + config_options_.invoke_prepare_options = false; + } bool RegisterTests(const std::string& arg) { #ifndef ROCKSDB_LITE config_options_.registry->AddLibrary("custom-tests", RegisterTestObjects, @@ -1047,6 +1105,49 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) { } } +#ifndef ROCKSDB_LITE +TEST_F(LoadCustomizableTest, LoadEncryptionProviderTest) { + std::shared_ptr result; + ASSERT_NOK( + EncryptionProvider::CreateFromString(config_options_, "Mock", &result)); + ASSERT_OK( + EncryptionProvider::CreateFromString(config_options_, "CTR", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "CTR"); + ASSERT_NOK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, "CTR://test", + &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "CTR"); + ASSERT_OK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + + if (RegisterTests("Test")) { + ASSERT_OK( + EncryptionProvider::CreateFromString(config_options_, "Mock", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "Mock"); + ASSERT_OK(EncryptionProvider::CreateFromString(config_options_, + "Mock://test", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "Mock"); + ASSERT_OK(result->ValidateOptions(DBOptions(), ColumnFamilyOptions())); + } +} + +TEST_F(LoadCustomizableTest, LoadEncryptionCipherTest) { + std::shared_ptr result; + ASSERT_NOK(BlockCipher::CreateFromString(config_options_, "Mock", &result)); + ASSERT_OK(BlockCipher::CreateFromString(config_options_, "ROT13", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "ROT13"); + if (RegisterTests("Test")) { + ASSERT_OK(BlockCipher::CreateFromString(config_options_, "Mock", &result)); + ASSERT_NE(result, nullptr); + ASSERT_STREQ(result->Name(), "Mock"); + } +} +#endif // !ROCKSDB_LITE + TEST_F(LoadCustomizableTest, LoadFlushBlockPolicyFactoryTest) { std::shared_ptr table; std::shared_ptr result;