andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)

Browse Source 
Summary:
This PR does a few things:

1.  The MockFileSystem class was split out from the MockEnv.  This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one).  The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.

2.  Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set.  To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).

3.  Updated the test framework to have a ROCKSDB_GTEST_SKIP macro.  This can be used to flag tests that are skipped.  Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).

I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV,  both, and neither under both MacOS and RedHat.  A few tests were disabled/skipped for the MEM/ENCRYPTED cases.  The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem.  (I will also push a change to disable those tests soon).  There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.

Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.

Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale.  I do not know how to do that, so if someone could write that job, it would be appreciated :)

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566

Reviewed By: zhichao-cao

Differential Revision: D24408980

Pulled By: jay-zhuang

fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
This commit is contained in:
mrambacher 2020-10-27 10:31:34 -07:00 committed by Facebook GitHub Bot
parent 6134ce6444
commit f35f7f2704
43 changed files with 969 additions and 726 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.circleci/config.yml
View File

@ -108,6 +108,26 @@ jobs:
- run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
build-linux-mem-env:
machine:
image: ubuntu-1604:202007-01
resource_class: 2xlarge
steps:
- pre-steps
- install-gflags
- run: MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
build-linux-encrypted-env:
machine:
image: ubuntu-1604:202007-01
resource_class: 2xlarge
steps:
- pre-steps
- install-gflags
- run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
- post-steps
build-linux-shared_lib-alt_namespace-status_checked:
machine:
image: ubuntu-1604:202007-01
@ -378,6 +398,12 @@ workflows:
build-linux:
jobs:
- build-linux
build-linux-mem-env:
jobs:
- build-linux-mem-env
build-linux-encrypted-env:
jobs:
- build-linux-encrypted-env
build-linux-shared_lib-alt_namespace-status_checked:
jobs:
- build-linux-shared_lib-alt_namespace-status_checked

19
db/blob/db_blob_basic_test.cc
View File

@ -18,7 +18,7 @@ class DBBlobBasicTest : public DBTestBase {
};
TEST_F(DBBlobBasicTest, GetBlob) {
Options options;
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = 0;
@ -45,7 +45,7 @@ TEST_F(DBBlobBasicTest, GetBlob) {
}
TEST_F(DBBlobBasicTest, GetBlob_CorruptIndex) {
Options options;
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = 0;
@ -70,7 +70,7 @@ TEST_F(DBBlobBasicTest, GetBlob_CorruptIndex) {
TEST_F(DBBlobBasicTest, GetBlob_InlinedTTLIndex) {
constexpr uint64_t min_blob_size = 10;
Options options;
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = min_blob_size;
@ -100,7 +100,7 @@ TEST_F(DBBlobBasicTest, GetBlob_InlinedTTLIndex) {
}
TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
Options options;
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = 0;
@ -132,11 +132,12 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
public testing::WithParamInterface<std::string> {
protected:
DBBlobBasicIOErrorTest()
: fault_injection_env_(Env::Default()), sync_point_(GetParam()) {}
DBBlobBasicIOErrorTest() : sync_point_(GetParam()) {
fault_injection_env_.reset(new FaultInjectionTestEnv(env_));
}
~DBBlobBasicIOErrorTest() { Close(); }
FaultInjectionTestEnv fault_injection_env_;
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env_;
std::string sync_point_;
};
@ -147,7 +148,7 @@ INSTANTIATE_TEST_CASE_P(DBBlobBasicTest, DBBlobBasicIOErrorTest,
TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) {
Options options;
options.env = &fault_injection_env_;
options.env = fault_injection_env_.get();
options.enable_blob_files = true;
options.min_blob_size = 0;
@ -161,7 +162,7 @@ TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) {
ASSERT_OK(Flush());
SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
fault_injection_env_.SetFilesystemActive(false,
fault_injection_env_->SetFilesystemActive(false,
Status::IOError(sync_point_));
});
SyncPoint::GetInstance()->EnableProcessing();

1
db/blob/db_blob_index_test.cc
View File

@ -103,6 +103,7 @@ class DBBlobIndexTest : public DBTestBase {
Options GetTestOptions() {
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.num_levels = 2;
options.disable_auto_compactions = true;

8
db/corruption_test.cc
View File

@ -184,7 +184,7 @@ class CorruptionTest : public testing::Test {
}
ASSERT_TRUE(!fname.empty()) << filetype;
test::CorruptFile(fname, offset, bytes_to_corrupt);
ASSERT_OK(test::CorruptFile(&env_, fname, offset, bytes_to_corrupt));
}
// corrupts exactly one file at level `level`. if no file found at level,
@ -194,7 +194,8 @@ class CorruptionTest : public testing::Test {
db_->GetLiveFilesMetaData(&metadata);
for (const auto& m : metadata) {
if (m.level == level) {
test::CorruptFile(dbname_ + "/" + m.name, offset, bytes_to_corrupt);
ASSERT_OK(test::CorruptFile(&env_, dbname_ + "/" + m.name, offset,
bytes_to_corrupt));
return;
}
}
@ -529,7 +530,8 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
ImmutableCFOptions(options_), kRangeDelBlock, &range_del_handle));
ASSERT_OK(TryReopen());
test::CorruptFile(filename, static_cast<int>(range_del_handle.offset()), 1);
ASSERT_OK(test::CorruptFile(&env_, filename,
static_cast<int>(range_del_handle.offset()), 1));
ASSERT_TRUE(TryReopen().IsCorruption());
}

15
db/db_basic_test.cc
View File

@ -12,6 +12,7 @@
#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "rocksdb/flush_block_policy.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/utilities/debug.h"
@ -409,8 +410,7 @@ TEST_F(DBBasicTest, CheckLock) {
Status s = DB::Open(options, dbname_, &localdb);
ASSERT_NOK(s);
#ifdef OS_LINUX
ASSERT_TRUE(s.ToString().find("lock hold by current process") !=
std::string::npos);
ASSERT_TRUE(s.ToString().find("lock ") != std::string::npos);
#endif // OS_LINUX
} while (ChangeCompactOptions());
}
@ -1875,6 +1875,7 @@ TEST_F(DBBasicTest, MultiGetStats) {
Options options;
options.create_if_missing = true;
options.disable_auto_compactions = true;
options.env = env_;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
BlockBasedTableOptions table_options;
table_options.block_size = 1;
@ -1884,7 +1885,7 @@ TEST_F(DBBasicTest, MultiGetStats) {
table_options.no_block_cache = true;
table_options.cache_index_and_filter_blocks = false;
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options);
int total_keys = 2000;
@ -2168,7 +2169,7 @@ TEST_F(DBBasicTest, MultiGetIOBufferOverrun) {
table_options.block_size = 16 * 1024;
ASSERT_TRUE(table_options.block_size >
BlockBasedTable::kMultiGetReadStackBufSize);
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options);
std::string zero_str(128, '\0');
@ -2549,7 +2550,7 @@ class DBBasicTestMultiGet : public DBTestBase {
table_options.block_cache_compressed = compressed_cache_;
table_options.flush_block_policy_factory.reset(
new MyFlushBlockPolicyFactory());
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
if (!compression_enabled_) {
options.compression = kNoCompression;
} else {
@ -2976,7 +2977,7 @@ class DeadlineFS : public FileSystemWrapper {
// or to simply delay but return success anyway. The latter mimics the
// behavior of PosixFileSystem, which does not enforce any timeout
explicit DeadlineFS(SpecialEnv* env, bool error_on_delay)
: FileSystemWrapper(FileSystem::Default()),
: FileSystemWrapper(env->GetFileSystem()),
deadline_(std::chrono::microseconds::zero()),
io_timeout_(std::chrono::microseconds::zero()),
env_(env),
@ -3151,7 +3152,7 @@ TEST_F(DBBasicTestMultiGetDeadline, MultiGetDeadlineExceeded) {
std::shared_ptr<Cache> cache = NewLRUCache(1048576);
BlockBasedTableOptions table_options;
table_options.block_cache = cache;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
options.env = env.get();
SetTimeElapseOnlySleepOnReopen(&options);
ReopenWithColumnFamilies(GetCFNames(), options);

3
db/db_bloom_filter_test.cc
View File

@ -1730,6 +1730,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterUpperBound) {
int using_full_builder = bfp_impl != BFP::kDeprecatedBlock;
Options options;
options.create_if_missing = true;
options.env = CurrentOptions().env;
options.prefix_extractor.reset(NewCappedPrefixTransform(4));
options.disable_auto_compactions = true;
options.statistics = CreateDBStatistics();
@ -1860,6 +1861,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
for (auto bfp_impl : BFP::kAllFixedImpls) {
int using_full_builder = bfp_impl != BFP::kDeprecatedBlock;
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
options.disable_auto_compactions = true;
@ -2052,6 +2054,7 @@ TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) {
TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
for (auto bfp_impl : BFP::kAllFixedImpls) {
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
options.disable_auto_compactions = true;

6
db/db_compaction_test.cc
View File

@ -3270,7 +3270,7 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
table_options.block_cache = NewLRUCache(64 << 20); // 64MB
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options);
@ -5023,7 +5023,7 @@ TEST_F(DBCompactionTest, ManualCompactionFailsInReadOnlyMode) {
// is in read-only mode. Verify it now at least returns, despite failing.
const int kNumL0Files = 4;
std::unique_ptr<FaultInjectionTestEnv> mock_env(
new FaultInjectionTestEnv(Env::Default()));
new FaultInjectionTestEnv(env_));
Options opts = CurrentOptions();
opts.disable_auto_compactions = true;
opts.env = mock_env.get();
@ -5250,7 +5250,7 @@ TEST_F(DBCompactionTest, ConsistencyFailTest2) {
options.level0_file_num_compaction_trigger = 2;
BlockBasedTableOptions bbto;
bbto.block_size = 400; // small block size
options.table_factory.reset(new BlockBasedTableFactory(bbto));
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
DestroyAndReopen(options);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(

17
db/db_encryption_test.cc
View File

@ -18,6 +18,13 @@ class DBEncryptionTest : public DBTestBase {
public:
DBEncryptionTest()
: DBTestBase("/db_encryption_test", /*env_do_fsync=*/true) {}
Env* GetTargetEnv() {
if (encrypted_env_ != nullptr) {
return (static_cast<EnvWrapper*>(encrypted_env_))->target();
} else {
return env_;
}
}
};
#ifndef ROCKSDB_LITE
@ -34,20 +41,20 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
auto status = env_->GetChildren(dbname_, &fileNames);
ASSERT_OK(status);
auto defaultEnv = Env::Default();
Env* target = GetTargetEnv();
int hits = 0;
for (auto it = fileNames.begin() ; it != fileNames.end(); ++it) {
if ((*it == "..") || (*it == ".")) {
if ((*it == "..") || (*it == ".") || (*it == "LOCK")) {
continue;
}
auto filePath = dbname_ + "/" + *it;
std::unique_ptr<SequentialFile> seqFile;
auto envOptions = EnvOptions(CurrentOptions());
status = defaultEnv->NewSequentialFile(filePath, &seqFile, envOptions);
status = target->NewSequentialFile(filePath, &seqFile, envOptions);
ASSERT_OK(status);
uint64_t fileSize;
status = defaultEnv->GetFileSize(filePath, &fileSize);
status = target->GetFileSize(filePath, &fileSize);
ASSERT_OK(status);
std::string scratch;
@ -85,7 +92,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
}
TEST_F(DBEncryptionTest, ReadEmptyFile) {
auto defaultEnv = Env::Default();
auto defaultEnv = GetTargetEnv();
// create empty file for reading it back in later
auto envOptions = EnvOptions(CurrentOptions());

1
db/db_flush_test.cc
View File

@ -450,6 +450,7 @@ TEST_F(DBFlushTest, FlushWithBlob) {
constexpr uint64_t min_blob_size = 10;
Options options;
options.env = CurrentOptions().env;
options.enable_blob_files = true;
options.min_blob_size = min_blob_size;
options.disable_auto_compactions = true;

2
db/db_impl/db_impl.cc
View File

@ -637,7 +637,7 @@ Status DBImpl::CloseHelper() {
if (immutable_db_options_.info_log && own_info_log_) {
Status s = immutable_db_options_.info_log->Close();
if (!s.ok() && ret.ok()) {
if (!s.ok() && !s.IsNotSupported() && ret.ok()) {
ret = s;
}
}

2
db/db_impl/db_secondary_test.cc
View File

@ -883,6 +883,7 @@ TEST_F(DBSecondaryTest, StartFromInconsistent) {
});
SyncPoint::GetInstance()->EnableProcessing();
Options options1;
options1.env = env_;
Status s = TryOpenSecondary(options1);
ASSERT_TRUE(s.IsCorruption());
}
@ -894,6 +895,7 @@ TEST_F(DBSecondaryTest, InconsistencyDuringCatchUp) {
ASSERT_OK(Flush());
Options options1;
options1.env = env_;
OpenSecondary(options1);
{

1
db/db_memtable_test.cc
View File

@ -316,6 +316,7 @@ TEST_F(DBMemTableTest, InsertWithHint) {
TEST_F(DBMemTableTest, ColumnFamilyId) {
// Verifies MemTableRepFactory is told the right column family id.
Options options;
options.env = CurrentOptions().env;
options.allow_concurrent_memtable_write = false;
options.create_if_missing = true;
options.memtable_factory.reset(new MockMemTableRepFactory());

8
db/db_options_test.cc
View File

@ -81,6 +81,7 @@ class DBOptionsTest : public DBTestBase {
TEST_F(DBOptionsTest, ImmutableTrackAndVerifyWalsInManifest) {
Options options;
options.env = env_;
options.track_and_verify_wals_in_manifest = true;
ImmutableDBOptions db_options(options);
@ -621,6 +622,7 @@ TEST_F(DBOptionsTest, MaxOpenFilesChange) {
TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
Options options;
options.env = CurrentOptions().env;
options.delayed_write_rate = 0;
Reopen(options);
ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
@ -632,6 +634,7 @@ TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) {
Options options;
options.env = CurrentOptions().env;
options.compaction_style = kCompactionStyleUniversal;
options.ttl = 0;
@ -661,6 +664,7 @@ TEST_F(DBOptionsTest, SanitizeUniversalTTLCompaction) {
TEST_F(DBOptionsTest, SanitizeTtlDefault) {
Options options;
options.env = CurrentOptions().env;
Reopen(options);
ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl);
@ -677,6 +681,7 @@ TEST_F(DBOptionsTest, SanitizeTtlDefault) {
TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) {
Options options;
options.compaction_style = kCompactionStyleFIFO;
options.env = CurrentOptions().env;
options.ttl = 0;
Reopen(options);
ASSERT_EQ(30 * 24 * 60 * 60, dbfull()->GetOptions().ttl);
@ -702,6 +707,7 @@ TEST_F(DBOptionsTest, SanitizeFIFOPeriodicCompaction) {
TEST_F(DBOptionsTest, SetFIFOCompactionOptions) {
Options options;
options.env = CurrentOptions().env;
options.compaction_style = kCompactionStyleFIFO;
options.write_buffer_size = 10 << 10; // 10KB
options.arena_block_size = 4096;
@ -841,6 +847,7 @@ TEST_F(DBOptionsTest, FIFOTtlBackwardCompatible) {
options.compaction_style = kCompactionStyleFIFO;
options.write_buffer_size = 10 << 10; // 10KB
options.create_if_missing = true;
options.env = CurrentOptions().env;
ASSERT_OK(TryReopen(options));
@ -894,6 +901,7 @@ TEST_F(DBOptionsTest, ChangeCompression) {
options.bottommost_compression = CompressionType::kNoCompression;
options.bottommost_compression_opts.level = 2;
options.bottommost_compression_opts.parallel_threads = 1;
options.env = CurrentOptions().env;
ASSERT_OK(TryReopen(options));

9
db/db_properties_test.cc
View File

@ -336,7 +336,7 @@ TEST_F(DBPropertiesTest, AggregatedTableProperties) {
table_options.filter_policy.reset(
NewBloomFilterPolicy(kBloomBitsPerKey, false));
table_options.block_size = 1024;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options);
@ -536,7 +536,7 @@ TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
table_options.filter_policy.reset(
NewBloomFilterPolicy(kBloomBitsPerKey, false));
table_options.block_size = 1024;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options);
@ -1414,7 +1414,7 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
}
TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
Options options;
Options options = CurrentOptions();
Reopen(options);
ASSERT_OK(Put("foo", "bar"));
ASSERT_OK(Delete("foo"));
@ -1524,6 +1524,7 @@ TEST_F(DBPropertiesTest, SstFilesSize) {
std::shared_ptr<TestListener> listener = std::make_shared<TestListener>();
Options options;
options.env = CurrentOptions().env;
options.disable_auto_compactions = true;
options.listeners.push_back(listener);
Reopen(options);
@ -1608,6 +1609,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) {
Options options;
uint64_t value;
options.env = CurrentOptions().env;
// Block cache properties are not available for tables other than
// block-based table.
options.table_factory.reset(NewPlainTableFactory());

3
db/db_sst_test.cc
View File

@ -141,6 +141,7 @@ TEST_F(DBSSTTest, SkipCheckingSSTFileSizesOnDBOpen) {
// Just open the DB with the option set to true and check that we don't crash.
Options options;
options.env = env_;
options.skip_checking_sst_file_sizes_on_db_open = true;
Reopen(options);
@ -519,7 +520,7 @@ TEST_P(DBWALTestWithParam, WALTrashCleanupOnOpen) {
bool fake_log_delete;
};
std::unique_ptr<MyEnv> env(new MyEnv(Env::Default()));
std::unique_ptr<MyEnv> env(new MyEnv(env_));
Destroy(last_options_);
env->set_fake_log_delete(true);

5
db/db_test.cc
View File

@ -2391,6 +2391,7 @@ TEST_F(DBTest, PurgeInfoLogs) {
Options options = CurrentOptions();
options.keep_log_file_num = 5;
options.create_if_missing = true;
options.env = env_;
for (int mode = 0; mode <= 1; mode++) {
if (mode == 1) {
options.db_log_dir = dbname_ + "_logs";
@ -3947,6 +3948,7 @@ TEST_F(DBTest, WriteSingleThreadEntry) {
TEST_F(DBTest, ConcurrentFlushWAL) {
const size_t cnt = 100;
Options options;
options.env = env_;
WriteOptions wopt;
ReadOptions ropt;
for (bool two_write_queues : {false, true}) {
@ -4615,6 +4617,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
Random rnd(301);
Options options;
options.env = env_;
options.create_if_missing = true;
options.db_write_buffer_size = 20480;
options.write_buffer_size = 20480;
@ -5017,6 +5020,7 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
Options options;
options.ttl = 0;
options.create_if_missing = true;
options.env = env_;
DestroyAndReopen(options);
// Initial defaults
@ -5078,6 +5082,7 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
TEST_F(DBTest, DynamicUniversalCompactionOptions) {
Options options;
options.create_if_missing = true;
options.env = env_;
DestroyAndReopen(options);
// Initial defaults

60
db/db_test2.cc
View File

@ -104,6 +104,7 @@ class TestReadOnlyWithCompressedCache
TEST_P(TestReadOnlyWithCompressedCache, ReadOnlyWithCompressedCache) {
if (use_mmap_ && !IsMemoryMappedAccessSupported()) {
ROCKSDB_GTEST_SKIP("Test requires MMAP support");
return;
}
ASSERT_OK(Put("foo", "bar"));
@ -291,7 +292,7 @@ TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) {
BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = true;
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options);
Put(1, "a", "begin");
@ -1344,7 +1345,7 @@ TEST_F(DBTest2, PresetCompressionDictLocality) {
options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry;
BlockBasedTableOptions table_options;
table_options.cache_index_and_filter_blocks = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options);
Random rnd(301);
@ -1470,7 +1471,7 @@ TEST_P(CompressionFailuresTest, CompressionFailures) {
BlockBasedTableOptions table_options;
table_options.block_size = 512;
table_options.verify_compression = true;
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
options.compression = compression_type_;
options.compression_opts.parallel_threads = compression_parallel_threads_;
@ -1808,7 +1809,7 @@ class PinL0IndexAndFilterBlocksTest
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options->table_factory.reset(new BlockBasedTableFactory(table_options));
options->table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, *options);
Put(1, "a", "begin");
@ -1848,7 +1849,7 @@ TEST_P(PinL0IndexAndFilterBlocksTest,
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
options.table_factory.reset(new BlockBasedTableFactory(table_options));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
CreateAndReopenWithCF({"pikachu"}, options);
ASSERT_OK(Put(1, "key", "val"));
@ -2485,26 +2486,30 @@ TEST_F(DBTest2, ReadAmpBitmapLiveInCacheAfterDBClose) {
{
const int kIdBufLen = 100;
char id_buf[kIdBufLen];
Status s = Status::NotSupported();
#ifndef OS_WIN
// You can't open a directory on windows using random access file
std::unique_ptr<RandomAccessFile> file;
ASSERT_OK(env_->NewRandomAccessFile(dbname_, &file, EnvOptions()));
s = env_->NewRandomAccessFile(dbname_, &file, EnvOptions());
if (s.ok()) {
if (file->GetUniqueId(id_buf, kIdBufLen) == 0) {
// fs holding db directory doesn't support getting a unique file id,
// this means that running this test will fail because lru_cache will load
// the blocks again regardless of them being already in the cache
// this means that running this test will fail because lru_cache will
// load the blocks again regardless of them being already in the cache
return;
}
#else
}
#endif
if (!s.ok()) {
std::unique_ptr<Directory> dir;
ASSERT_OK(env_->NewDirectory(dbname_, &dir));
if (dir->GetUniqueId(id_buf, kIdBufLen) == 0) {
// fs holding db directory doesn't support getting a unique file id,
// this means that running this test will fail because lru_cache will load
// the blocks again regardless of them being already in the cache
// this means that running this test will fail because lru_cache will
// load the blocks again regardless of them being already in the cache
return;
}
#endif
}
}
uint32_t bytes_per_bit[2] = {1, 16};
for (size_t k = 0; k < 2; k++) {
@ -3297,7 +3302,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
BlockBasedTableOptions bbto;
bbto.block_size = 16384;
bbto.no_block_cache = true;
options.table_factory.reset(new BlockBasedTableFactory(bbto));
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
DestroyAndReopen(options);
for (int i = 0; i < kNumL0Files; ++i) {
@ -3342,6 +3347,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
// is on levels higher than the new num_levels.
TEST_F(DBTest2, ReduceLevel) {
Options options;
options.env = env_;
options.disable_auto_compactions = true;
options.num_levels = 7;
Reopen(options);
@ -3370,6 +3376,7 @@ TEST_F(DBTest2, ReadCallbackTest) {
Options options;
options.disable_auto_compactions = true;
options.num_levels = 7;
options.env = env_;
Reopen(options);
std::vector<const Snapshot*> snapshots;
// Try to create a db with multiple layers and a memtable
@ -3629,7 +3636,9 @@ TEST_F(DBTest2, TraceAndReplay) {
column_families.push_back(
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
std::vector<ColumnFamilyHandle*> handles;
ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
DBOptions db_opts;
db_opts.env = env_;
ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2));
env_->SleepForMicroseconds(100);
// Verify that the keys don't already exist
@ -3704,7 +3713,9 @@ TEST_F(DBTest2, TraceWithLimit) {
column_families.push_back(
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
std::vector<ColumnFamilyHandle*> handles;
ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
DBOptions db_opts;
db_opts.env = env_;
ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2));
env_->SleepForMicroseconds(100);
// Verify that the keys don't already exist
@ -3772,7 +3783,9 @@ TEST_F(DBTest2, TraceWithSampling) {
column_families.push_back(
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
std::vector<ColumnFamilyHandle*> handles;
ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
DBOptions db_opts;
db_opts.env = env_;
ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2));
env_->SleepForMicroseconds(100);
ASSERT_TRUE(db2->Get(ro, handles[0], "a", &value).IsNotFound());
@ -3872,7 +3885,9 @@ TEST_F(DBTest2, TraceWithFilter) {
column_families.push_back(
ColumnFamilyDescriptor("pikachu", ColumnFamilyOptions()));
std::vector<ColumnFamilyHandle*> handles;
ASSERT_OK(DB::Open(DBOptions(), dbname2, column_families, &handles, &db2));
DBOptions db_opts;
db_opts.env = env_;
ASSERT_OK(DB::Open(db_opts, dbname2, column_families, &handles, &db2));
env_->SleepForMicroseconds(100);
// Verify that the keys don't already exist
@ -3918,7 +3933,7 @@ TEST_F(DBTest2, TraceWithFilter) {
handles.clear();
DB* db3 = nullptr;
ASSERT_OK(DB::Open(DBOptions(), dbname3, column_families, &handles, &db3));
ASSERT_OK(DB::Open(db_opts, dbname3, column_families, &handles, &db3));
env_->SleepForMicroseconds(100);
// Verify that the keys don't already exist
@ -3974,6 +3989,11 @@ TEST_F(DBTest2, TraceWithFilter) {
TEST_F(DBTest2, PinnableSliceAndMmapReads) {
Options options = CurrentOptions();
options.env = env_;
if (options.env != Env::Default()) {
ROCKSDB_GTEST_SKIP("Test requires default environment");
return;
}
options.allow_mmap_reads = true;
options.max_open_files = 100;
options.compression = kNoCompression;
@ -4026,7 +4046,7 @@ TEST_F(DBTest2, DISABLED_IteratorPinnedMemory) {
bbto.cache_index_and_filter_blocks = false;
bbto.block_cache = NewLRUCache(100000);
bbto.block_size = 400; // small block size
options.table_factory.reset(new BlockBasedTableFactory(bbto));
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
Reopen(options);
Random rnd(301);
@ -4252,6 +4272,7 @@ TEST_F(DBTest2, TestCompactFiles) {
SyncPoint::GetInstance()->EnableProcessing();
Options options;
options.env = env_;
options.num_levels = 2;
options.disable_auto_compactions = true;
Reopen(options);
@ -4807,6 +4828,7 @@ TEST_F(DBTest2, BlockBasedTablePrefixIndexSeekForPrev) {
TEST_F(DBTest2, PartitionedIndexPrefetchFailure) {
Options options = last_options_;
options.env = env_;
options.max_open_files = 20;
BlockBasedTableOptions bbto;
bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;

15
db/db_test_util.cc
View File

@ -331,7 +331,7 @@ Options DBTestBase::CurrentOptions(
return GetOptions(option_config_, default_options, options_override);
}
Options DBTestBase::GetDefaultOptions() {
Options DBTestBase::GetDefaultOptions() const {
Options options;
options.write_buffer_size = 4090 * 4096;
options.target_file_size_base = 2 * 1024 * 1024;
@ -339,6 +339,7 @@ Options DBTestBase::GetDefaultOptions() {
options.max_open_files = 5000;
options.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
options.compaction_pri = CompactionPri::kByCompensatedSize;
options.env = env_;
return options;
}
@ -367,28 +368,28 @@ Options DBTestBase::GetOptions(
options.unordered_write = false;
break;
case kPlainTableFirstBytePrefix:
options.table_factory.reset(new PlainTableFactory());
options.table_factory.reset(NewPlainTableFactory());
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
options.allow_mmap_reads = can_allow_mmap;
options.max_sequential_skip_in_iterations = 999999;
set_block_based_table_factory = false;
break;
case kPlainTableCappedPrefix:
options.table_factory.reset(new PlainTableFactory());
options.table_factory.reset(NewPlainTableFactory());
options.prefix_extractor.reset(NewCappedPrefixTransform(8));
options.allow_mmap_reads = can_allow_mmap;
options.max_sequential_skip_in_iterations = 999999;
set_block_based_table_factory = false;
break;
case kPlainTableCappedPrefixNonMmap:
options.table_factory.reset(new PlainTableFactory());
options.table_factory.reset(NewPlainTableFactory());
options.prefix_extractor.reset(NewCappedPrefixTransform(8));
options.allow_mmap_reads = false;
options.max_sequential_skip_in_iterations = 999999;
set_block_based_table_factory = false;
break;
case kPlainTableAllBytesPrefix:
options.table_factory.reset(new PlainTableFactory());
options.table_factory.reset(NewPlainTableFactory());
options.prefix_extractor.reset(NewNoopTransform());
options.allow_mmap_reads = can_allow_mmap;
options.max_sequential_skip_in_iterations = 999999;
@ -704,9 +705,9 @@ Status DBTestBase::TryReopen(const Options& options) {
// Note: operator= is an unsafe approach here since it destructs
// std::shared_ptr in the same order of their creation, in contrast to
// destructors which destructs them in the opposite order of creation. One
// particular problme is that the cache destructor might invoke callback
// particular problem is that the cache destructor might invoke callback
// functions that use Option members such as statistics. To work around this
// problem, we manually call destructor of table_facotry which eventually
// problem, we manually call destructor of table_factory which eventually
// clears the block cache.
last_options_ = options;
MaybeInstallTimeElapseOnlySleep(options);

9
db/db_test_util.h
View File

@ -949,10 +949,13 @@ class DBTestBase : public testing::Test {
const anon::OptionsOverride& options_override =
anon::OptionsOverride()) const;
static Options GetDefaultOptions();
Options GetDefaultOptions() const;
Options GetOptions(int option_config,
const Options& default_options = GetDefaultOptions(),
Options GetOptions(int option_config) const {
return GetOptions(option_config, GetDefaultOptions());
}
Options GetOptions(int option_config, const Options& default_options,
const anon::OptionsOverride& options_override =
anon::OptionsOverride()) const;

2
db/db_universal_compaction_test.cc
View File

@ -2129,7 +2129,7 @@ TEST_F(DBTestUniversalCompaction2, IngestBehind) {
TEST_F(DBTestUniversalCompaction2, PeriodicCompactionDefault) {
Options options;
options.compaction_style = kCompactionStyleUniversal;
options.env = env_;
KeepFilterFactory* filter = new KeepFilterFactory(true);
options.compaction_filter_factory.reset(filter);
Reopen(options);

34
db/db_wal_test.cc
View File

@ -1181,32 +1181,13 @@ class RecoveryTestHelper {
test->Close();
#endif
if (trunc) {
ASSERT_EQ(0, truncate(fname.c_str(), static_cast<int64_t>(size * off)));
ASSERT_OK(
test::TruncateFile(env, fname, static_cast<uint64_t>(size * off)));
} else {
InduceCorruption(fname, static_cast<size_t>(size * off + 8),
static_cast<size_t>(size * len));
ASSERT_OK(test::CorruptFile(env, fname, static_cast<int>(size * off + 8),
static_cast<int>(size * len), false));
}
}
// Overwrite data with 'a' from offset for length len
static void InduceCorruption(const std::string& filename, size_t offset,
size_t len) {
ASSERT_GT(len, 0U);
int fd = open(filename.c_str(), O_RDWR);
// On windows long is 32-bit
ASSERT_LE(offset, std::numeric_limits<long>::max());
ASSERT_GT(fd, 0);
ASSERT_EQ(offset, lseek(fd, static_cast<long>(offset), SEEK_SET));
void* buf = alloca(len);
memset(buf, 'b', len);
ASSERT_EQ(len, write(fd, buf, static_cast<unsigned int>(len)));
close(fd);
}
};
class DBWALTestWithParams
@ -1328,8 +1309,7 @@ TEST_F(DBWALTest, kPointInTimeRecoveryCFConsistency) {
ASSERT_OK(Put(1, "key3", "val3"));
// Corrupt WAL at location of key3
RecoveryTestHelper::InduceCorruption(
fname, static_cast<size_t>(offset_to_corrupt), static_cast<size_t>(4));
test::CorruptFile(env, fname, static_cast<int>(offset_to_corrupt), 4, false);
ASSERT_OK(Put(2, "key4", "val4"));
ASSERT_OK(Put(1, "key5", "val5"));
Flush(2);
@ -1717,6 +1697,10 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithoutFlush) {
constexpr size_t kKB = 1024;
Options options = CurrentOptions();
options.avoid_flush_during_recovery = true;
if (options.env != Env::Default()) {
ROCKSDB_GTEST_SKIP("Test requires default environment");
return;
}
// Test fallocate support of running file system.
// Skip this test if fallocate is not supported.
std::string fname_test_fallocate = dbname_ + "/preallocate_testfile";

6
db/db_write_test.cc
View File

@ -260,7 +260,7 @@ TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
constexpr int kNumThreads = 5;
std::unique_ptr<FaultInjectionTestEnv> mock_env(
new FaultInjectionTestEnv(Env::Default()));
new FaultInjectionTestEnv(env_));
Options options = GetOptions();
options.env = mock_env.get();
Reopen(options);
@ -329,7 +329,7 @@ TEST_P(DBWriteTest, ManualWalFlushInEffect) {
TEST_P(DBWriteTest, IOErrorOnWALWriteTriggersReadOnlyMode) {
std::unique_ptr<FaultInjectionTestEnv> mock_env(
new FaultInjectionTestEnv(Env::Default()));
new FaultInjectionTestEnv(env_));
Options options = GetOptions();
options.env = mock_env.get();
Reopen(options);
@ -361,7 +361,7 @@ TEST_P(DBWriteTest, IOErrorOnWALWriteTriggersReadOnlyMode) {
TEST_P(DBWriteTest, IOErrorOnSwitchMemtable) {
Random rnd(301);
std::unique_ptr<FaultInjectionTestEnv> mock_env(
new FaultInjectionTestEnv(Env::Default()));
new FaultInjectionTestEnv(env_));
Options options = GetOptions();
options.env = mock_env.get();
options.writable_file_max_buffer_size = 4 * 1024 * 1024;

387
db/error_handler_fs_test.cc
View File

File diff suppressed because it is too large Load Diff

6
db/external_sst_file_basic_test.cc
View File

@ -24,7 +24,7 @@ class ExternalSSTFileBasicTest
ExternalSSTFileBasicTest()
: DBTestBase("/external_sst_file_basic_test", /*env_do_fsync=*/true) {
sst_files_dir_ = dbname_ + "/sst_files/";
fault_injection_test_env_.reset(new FaultInjectionTestEnv(Env::Default()));
fault_injection_test_env_.reset(new FaultInjectionTestEnv(env_));
DestroyAndRecreateExternalSSTFilesDir();
}
@ -1109,6 +1109,7 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) {
}
Options sst_file_writer_options;
sst_file_writer_options.env = env_;
std::unique_ptr<SstFileWriter> sst_file_writer(
new SstFileWriter(EnvOptions(), sst_file_writer_options));
std::string file_name =
@ -1137,11 +1138,12 @@ TEST_F(ExternalSSTFileBasicTest, SyncFailure) {
TEST_F(ExternalSSTFileBasicTest, VerifyChecksumReadahead) {
Options options;
options.create_if_missing = true;
SpecialEnv senv(Env::Default());
SpecialEnv senv(env_);
options.env = &senv;
DestroyAndReopen(options);
Options sst_file_writer_options;
sst_file_writer_options.env = env_;
std::unique_ptr<SstFileWriter> sst_file_writer(
new SstFileWriter(EnvOptions(), sst_file_writer_options));
std::string file_name = sst_files_dir_ + "verify_checksum_readahead_test.sst";

2
db/import_column_family_test.cc
View File

@ -16,8 +16,8 @@ class ImportColumnFamilyTest : public DBTestBase {
ImportColumnFamilyTest()
: DBTestBase("/import_column_family_test", /*env_do_fsync=*/true) {
sst_files_dir_ = dbname_ + "/sst_files/";
DestroyAndRecreateExternalSSTFilesDir();
export_files_dir_ = test::PerThreadDBPath(env_, "export");
DestroyAndRecreateExternalSSTFilesDir();
import_cfh_ = nullptr;
import_cfh2_ = nullptr;
metadata_ptr_ = nullptr;

20
db/listener_test.cc
View File

@ -676,7 +676,7 @@ class TableFileCreationListener : public EventListener {
public:
class TestEnv : public EnvWrapper {
public:
TestEnv() : EnvWrapper(Env::Default()) {}
explicit TestEnv(Env* t) : EnvWrapper(t) {}
void SetStatus(Status s) { status_ = s; }
@ -688,7 +688,7 @@ class TableFileCreationListener : public EventListener {
return status_;
}
}
return Env::Default()->NewWritableFile(fname, result, options);
return target()->NewWritableFile(fname, result, options);
}
private:
@ -766,7 +766,6 @@ class TableFileCreationListener : public EventListener {
}
}
TestEnv test_env;
int started_[2];
int finished_[2];
int failure_[2];
@ -775,9 +774,11 @@ class TableFileCreationListener : public EventListener {
TEST_F(EventListenerTest, TableFileCreationListenersTest) {
auto listener = std::make_shared<TableFileCreationListener>();
Options options;
std::unique_ptr<TableFileCreationListener::TestEnv> test_env(
new TableFileCreationListener::TestEnv(CurrentOptions().env));
options.create_if_missing = true;
options.listeners.push_back(listener);
options.env = &listener->test_env;
options.env = test_env.get();
DestroyAndReopen(options);
ASSERT_OK(Put("foo", "aaa"));
@ -785,13 +786,12 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) {
ASSERT_OK(Flush());
dbfull()->TEST_WaitForFlushMemTable();
listener->CheckAndResetCounters(1, 1, 0, 0, 0, 0);
ASSERT_OK(Put("foo", "aaa1"));
ASSERT_OK(Put("bar", "bbb1"));
listener->test_env.SetStatus(Status::NotSupported("not supported"));
test_env->SetStatus(Status::NotSupported("not supported"));
ASSERT_NOK(Flush());
listener->CheckAndResetCounters(1, 1, 1, 0, 0, 0);
listener->test_env.SetStatus(Status::OK());
test_env->SetStatus(Status::OK());
Reopen(options);
ASSERT_OK(Put("foo", "aaa2"));
@ -809,10 +809,11 @@ TEST_F(EventListenerTest, TableFileCreationListenersTest) {
ASSERT_OK(Put("foo", "aaa3"));
ASSERT_OK(Put("bar", "bbb3"));
ASSERT_OK(Flush());
listener->test_env.SetStatus(Status::NotSupported("not supported"));
test_env->SetStatus(Status::NotSupported("not supported"));
dbfull()->CompactRange(CompactRangeOptions(), &kRangeStart, &kRangeEnd);
dbfull()->TEST_WaitForCompact();
listener->CheckAndResetCounters(1, 1, 0, 1, 1, 1);
Close();
}
class MemTableSealedListener : public EventListener {
@ -833,6 +834,7 @@ public:
TEST_F(EventListenerTest, MemTableSealedListenerTest) {
auto listener = std::make_shared<MemTableSealedListener>();
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.listeners.push_back(listener);
DestroyAndReopen(options);
@ -1066,7 +1068,7 @@ TEST_F(EventListenerTest, OnFileOperationTest) {
TestFileOperationListener* listener = new TestFileOperationListener();
options.listeners.emplace_back(listener);
options.use_direct_io_for_flush_and_compaction = true;
options.use_direct_io_for_flush_and_compaction = false;
Status s = TryReopen(options);
if (s.IsInvalidArgument()) {
options.use_direct_io_for_flush_and_compaction = false;

4
db/perf_context_test.cc
View File

@ -818,9 +818,7 @@ TEST_F(PerfContextTest, PerfContextByLevelGetSet) {
TEST_F(PerfContextTest, CPUTimer) {
if (Env::Default()->NowCPUNanos() == 0) {
// TODO: This should be a GTEST_SKIP when the embedded gtest is updated
// to 1.10 or higher.
GTEST_SUCCESS_("Skipped on target without NowCPUNanos support");
ROCKSDB_GTEST_SKIP("Target without NowCPUNanos support");
return;
}

5
db/periodic_work_scheduler_test.cc
View File

@ -13,8 +13,9 @@ namespace ROCKSDB_NAMESPACE {
class PeriodicWorkSchedulerTest : public DBTestBase {
public:
PeriodicWorkSchedulerTest()
: DBTestBase("/periodic_work_scheduler_test", /*env_do_fsync=*/true),
mock_env_(new MockTimeEnv(Env::Default())) {}
: DBTestBase("/periodic_work_scheduler_test", /*env_do_fsync=*/true) {
mock_env_.reset(new MockTimeEnv(env_));
}
protected:
std::unique_ptr<MockTimeEnv> mock_env_;

49
db/version_set_test.cc
View File

@ -8,10 +8,11 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/version_set.h"
#include "db/db_impl/db_impl.h"
#include "db/log_writer.h"
#include "env/mock_env.h"
#include "logging/logging.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/mock_table.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
@ -692,10 +693,7 @@ class VersionSetTestBase {
int num_initial_edits_;
explicit VersionSetTestBase(const std::string& name)
: mem_env_(nullptr),
env_(nullptr),
env_guard_(),
fs_(),
: env_(nullptr),
dbname_(test::PerThreadDBPath(name)),
options_(),
db_options_(options_),
@ -707,25 +705,26 @@ class VersionSetTestBase {
shutting_down_(false),
mock_table_factory_(std::make_shared<mock::MockTableFactory>()) {
const char* test_env_uri = getenv("TEST_ENV_URI");
Env* base_env = nullptr;
if (test_env_uri) {
Status s = Env::LoadEnv(test_env_uri, &base_env, &env_guard_);
Status s = Env::LoadEnv(test_env_uri, &env_, &env_guard_);
EXPECT_OK(s);
EXPECT_NE(Env::Default(), base_env);
} else if (getenv("MEM_ENV")) {
env_guard_.reset(NewMemEnv(Env::Default()));
env_ = env_guard_.get();
} else {
base_env = Env::Default();
env_ = Env::Default();
}
EXPECT_NE(nullptr, base_env);
if (getenv("MEM_ENV")) {
mem_env_ = new MockEnv(base_env);
}
env_ = mem_env_ ? mem_env_ : base_env;
EXPECT_NE(nullptr, env_);
fs_ = std::make_shared<LegacyFileSystemWrapper>(env_);
EXPECT_OK(env_->CreateDirIfMissing(dbname_));
fs_ = env_->GetFileSystem();
EXPECT_OK(fs_->CreateDirIfMissing(dbname_, IOOptions(), nullptr));
options_.env = env_;
db_options_.env = env_;
db_options_.fs = fs_;
immutable_cf_options_.env = env_;
immutable_cf_options_.fs = fs_.get();
versions_.reset(
new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(),
&write_buffer_manager_, &write_controller_,
@ -745,10 +744,6 @@ class VersionSetTestBase {
options.env = env_;
EXPECT_OK(DestroyDB(dbname_, options));
}
if (mem_env_) {
delete mem_env_;
mem_env_ = nullptr;
}
}
protected:
@ -760,7 +755,9 @@ class VersionSetTestBase {
assert(log_writer != nullptr);
VersionEdit new_db;
if (db_options_.write_dbid_to_manifest) {
std::unique_ptr<DBImpl> impl(new DBImpl(DBOptions(), dbname_));
DBOptions tmp_db_options;
tmp_db_options.env = env_;
std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
std::string db_id;
impl->GetDbIdentityFromIdentityFile(&db_id);
new_db.SetDBId(db_id);
@ -873,7 +870,7 @@ class VersionSetTestBase {
mutex_.Unlock();
}
MockEnv* mem_env_;
Env* mem_env_;
Env* env_;
std::shared_ptr<Env> env_guard_;
std::shared_ptr<FileSystem> fs_;
@ -2216,7 +2213,9 @@ class VersionSetTestEmptyDb
assert(nullptr != log_writer);
VersionEdit new_db;
if (db_options_.write_dbid_to_manifest) {
std::unique_ptr<DBImpl> impl(new DBImpl(DBOptions(), dbname_));
DBOptions tmp_db_options;
tmp_db_options.env = env_;
std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
std::string db_id;
impl->GetDbIdentityFromIdentityFile(&db_id);
new_db.SetDBId(db_id);
@ -2541,7 +2540,9 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
log_writer->reset(new log::Writer(std::move(file_writer), 0, false));
VersionEdit new_db;
if (db_options_.write_dbid_to_manifest) {
std::unique_ptr<DBImpl> impl(new DBImpl(DBOptions(), dbname_));
DBOptions tmp_db_options;
tmp_db_options.env = env_;
std::unique_ptr<DBImpl> impl(new DBImpl(tmp_db_options, dbname_));
std::string db_id;
impl->GetDbIdentityFromIdentityFile(&db_id);
new_db.SetDBId(db_id);

4
env/env_basic_test.cc vendored
View File

@ -228,8 +228,8 @@ TEST_P(EnvBasicTestWithParam, Basics) {
ASSERT_EQ(Status::NotFound(), env_->FileExists(test_dir_ + "/g"));
ASSERT_OK(env_->GetChildren(test_dir_, &children));
ASSERT_EQ(0U, children.size());
ASSERT_TRUE(
env_->GetChildren(test_dir_ + "/non_existent", &children).IsNotFound());
Status s = env_->GetChildren(test_dir_ + "/non_existent", &children);
ASSERT_TRUE(s.IsNotFound());
}
TEST_P(EnvBasicTestWithParam, ReadWrite) {

683
env/mock_env.cc vendored
View File

File diff suppressed because it is too large Load Diff

82
env/mock_env.h vendored
View File

@ -12,88 +12,17 @@
#include <map>
#include <string>
#include <vector>
#include "env/composite_env_wrapper.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "port/port.h"
#include "util/mutexlock.h"
namespace ROCKSDB_NAMESPACE {
class MemFile;
class MockEnv : public EnvWrapper {
class MockEnv : public CompositeEnvWrapper {
public:
explicit MockEnv(Env* base_env);
~MockEnv() override;
// Partial implementation of the Env interface.
Status RegisterDbPaths(const std::vector<std::string>& /*paths*/) override {
return Status::OK();
}
Status UnregisterDbPaths(const std::vector<std::string>& /*paths*/) override {
return Status::OK();
}
Status NewSequentialFile(const std::string& fname,
std::unique_ptr<SequentialFile>* result,
const EnvOptions& soptions) override;
Status NewRandomAccessFile(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result,
const EnvOptions& soptions) override;
Status NewRandomRWFile(const std::string& fname,
std::unique_ptr<RandomRWFile>* result,
const EnvOptions& options) override;
Status ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
std::unique_ptr<WritableFile>* result,
const EnvOptions& options) override;
Status NewWritableFile(const std::string& fname,
std::unique_ptr<WritableFile>* result,
const EnvOptions& env_options) override;
Status NewDirectory(const std::string& name,
std::unique_ptr<Directory>* result) override;
Status FileExists(const std::string& fname) override;
Status GetChildren(const std::string& dir,
std::vector<std::string>* result) override;
void DeleteFileInternal(const std::string& fname);
Status DeleteFile(const std::string& fname) override;
Status Truncate(const std::string& fname, size_t size) override;
Status CreateDir(const std::string& dirname) override;
Status CreateDirIfMissing(const std::string& dirname) override;
Status DeleteDir(const std::string& dirname) override;
Status GetFileSize(const std::string& fname, uint64_t* file_size) override;
Status GetFileModificationTime(const std::string& fname,
uint64_t* time) override;
Status RenameFile(const std::string& src, const std::string& target) override;
Status LinkFile(const std::string& src, const std::string& target) override;
Status NewLogger(const std::string& fname,
std::shared_ptr<Logger>* result) override;
Status LockFile(const std::string& fname, FileLock** flock) override;
Status UnlockFile(FileLock* flock) override;
Status GetTestDirectory(std::string* path) override;
// Results of these can be affected by FakeSleepForMicroseconds()
Status GetCurrentTime(int64_t* unix_time) override;
uint64_t NowMicros() override;
@ -106,11 +35,6 @@ class MockEnv : public EnvWrapper {
void FakeSleepForMicroseconds(int64_t micros);
private:
// Map from filenames to MemFile objects, representing a simple file system.
typedef std::map<std::string, MemFile*> FileSystem;
port::Mutex mutex_;
FileSystem file_map_; // Protected by mutex_.
std::atomic<int64_t> fake_sleep_micros_;
};

2
file/file_util.cc
View File

@ -211,6 +211,8 @@ Status DestroyDir(Env* env, const std::string& dir) {
} else {
s = env->DeleteFile(path);
}
} else if (s.IsNotSupported()) {
s = Status::OK();
}
if (!s.ok()) {
// IsDirectory, etc. might not report NotFound

3
file/filename.cc
View File

@ -184,7 +184,8 @@ InfoLogPrefix::InfoLogPrefix(bool has_log_dir,
snprintf(buf, sizeof(buf), kInfoLogPrefix);
prefix = Slice(buf, sizeof(kInfoLogPrefix) - 1);
} else {
size_t len = GetInfoLogPrefix(db_absolute_path, buf, sizeof(buf));
size_t len =
GetInfoLogPrefix(NormalizePath(db_absolute_path), buf, sizeof(buf));
prefix = Slice(buf, len);
}
}

12
file/prefetch_test.cc
View File

@ -25,7 +25,7 @@ class MockRandomAccessFile : public FSRandomAccessFileWrapper {
prefetch_count_.fetch_add(1);
return target()->Prefetch(offset, n, options, dbg);
} else {
return IOStatus::NotSupported();
return IOStatus::NotSupported("Prefetch not supported");
}
}
@ -37,9 +37,9 @@ class MockRandomAccessFile : public FSRandomAccessFileWrapper {
class MockFS : public FileSystemWrapper {
public:
explicit MockFS(bool support_prefetch)
: FileSystemWrapper(FileSystem::Default()),
support_prefetch_(support_prefetch) {}
explicit MockFS(const std::shared_ptr<FileSystem>& wrapped,
bool support_prefetch)
: FileSystemWrapper(wrapped), support_prefetch_(support_prefetch) {}
IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& opts,
@ -79,9 +79,9 @@ TEST_P(PrefetchTest, Basic) {
// Second param is if directIO is enabled or not
bool use_direct_io = std::get<1>(GetParam());
const int kNumKeys = 1100;
std::shared_ptr<MockFS> fs = std::make_shared<MockFS>(support_prefetch);
std::shared_ptr<MockFS> fs =
std::make_shared<MockFS>(env_->GetFileSystem(), support_prefetch);
std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, fs));
Options options = CurrentOptions();
options.write_buffer_size = 1024;

12
include/rocksdb/file_system.h
View File

@ -262,7 +262,7 @@ class FileSystem {
virtual IOStatus ReopenWritableFile(
const std::string& /*fname*/, const FileOptions& /*options*/,
std::unique_ptr<FSWritableFile>* /*result*/, IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
return IOStatus::NotSupported("ReopenWritableFile");
}
// Reuse an existing file by renaming it and opening it as writable.
@ -523,7 +523,7 @@ class FileSystem {
const IOOptions& /*options*/,
uint64_t* /*diskfree*/,
IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
return IOStatus::NotSupported("GetFreeSpace");
}
virtual IOStatus IsDirectory(const std::string& /*path*/,
@ -584,7 +584,7 @@ class FSSequentialFile {
const IOOptions& /*options*/,
Slice* /*result*/, char* /*scratch*/,
IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
return IOStatus::NotSupported("PositionedRead");
}
// If you're adding methods here, remember to add them to
@ -638,7 +638,7 @@ class FSRandomAccessFile {
virtual IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
return IOStatus::NotSupported("Prefetch");
}
// Read a bunch of blocks as described by reqs. The blocks can
@ -770,7 +770,7 @@ class FSWritableFile {
uint64_t /* offset */,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
return IOStatus::NotSupported("PositionedAppend");
}
// EXPERIMENTAL / CURRENTLY UNUSED
@ -782,7 +782,7 @@ class FSWritableFile {
const IOOptions& /*options*/,
const DataVerificationInfo& /* verification_info */,
IODebugContext* /*dbg*/) {
return IOStatus::NotSupported();
return IOStatus::NotSupported("PositionedAppend");
}
// Truncate is necessary to trim the file to the correct size

5
monitoring/stats_history_test.cc
View File

@ -32,8 +32,9 @@ namespace ROCKSDB_NAMESPACE {
class StatsHistoryTest : public DBTestBase {
public:
StatsHistoryTest()
: DBTestBase("/stats_history_test", /*env_do_fsync=*/true),
mock_env_(new MockTimeEnv(Env::Default())) {}
: DBTestBase("/stats_history_test", /*env_do_fsync=*/true) {
mock_env_.reset(new MockTimeEnv(env_));
}
protected:
std::unique_ptr<MockTimeEnv> mock_env_;

3
table/block_based/block_based_table_reader_test.cc
View File

@ -299,7 +299,8 @@ TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) {
table.reset();
// Corrupt the block pointed to by handle
test::CorruptFile(Path(table_name), static_cast<int>(handle.offset()), 128);
ASSERT_OK(test::CorruptFile(options.env, Path(table_name),
static_cast<int>(handle.offset()), 128));
NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table);
Status s = table->VerifyChecksum(ReadOptions(),

7
test_util/testharness.h
View File

@ -15,6 +15,13 @@
#include <gtest/gtest.h>
#endif
// If GTEST_SKIP is available, use it. Otherwise, define skip as success
#ifdef GTEST_SKIP_
#define ROCKSDB_GTEST_SKIP(m) GTEST_SKIP_(m)
#else
#define ROCKSDB_GTEST_SKIP(m) GTEST_SUCCESS_("SKIPPED: " m)
#endif
#include <string>
#include "rocksdb/env.h"

66
test_util/testutil.cc
View File

@ -486,47 +486,63 @@ size_t GetLinesCount(const std::string& fname, const std::string& pattern) {
return count;
}
void CorruptFile(const std::string& fname, int offset, int bytes_to_corrupt) {
struct stat sbuf;
if (stat(fname.c_str(), &sbuf) != 0) {
// strerror is not thread-safe so should not be used in the "passing" path
// of unit tests (sometimes parallelized) but is OK here where test fails
const char* msg = strerror(errno);
fprintf(stderr, "%s:%s\n", fname.c_str(), msg);
assert(false);
}
if (offset < 0) {
Status CorruptFile(Env* env, const std::string& fname, int offset,
int bytes_to_corrupt, bool verify_checksum /*=true*/) {
uint64_t size;
Status s = env->GetFileSize(fname, &size);
if (!s.ok()) {
return s;
} else if (offset < 0) {
// Relative to end of file; make it absolute
if (-offset > sbuf.st_size) {
if (-offset > static_cast<int>(size)) {
offset = 0;
} else {
offset = static_cast<int>(sbuf.st_size + offset);
offset = static_cast<int>(size + offset);
}
}
if (offset > sbuf.st_size) {
offset = static_cast<int>(sbuf.st_size);
if (offset > static_cast<int>(size)) {
offset = static_cast<int>(size);
}
if (offset + bytes_to_corrupt > sbuf.st_size) {
bytes_to_corrupt = static_cast<int>(sbuf.st_size - offset);
if (offset + bytes_to_corrupt > static_cast<int>(size)) {
bytes_to_corrupt = static_cast<int>(size - offset);
}
// Do it
std::string contents;
Status s = ReadFileToString(Env::Default(), fname, &contents);
assert(s.ok());
s = ReadFileToString(env, fname, &contents);
if (s.ok()) {
for (int i = 0; i < bytes_to_corrupt; i++) {
contents[i + offset] ^= 0x80;
}
s = WriteStringToFile(Env::Default(), contents, fname);
assert(s.ok());
Options options;
EnvOptions env_options;
s = WriteStringToFile(env, contents, fname);
}
if (s.ok() && verify_checksum) {
#ifndef ROCKSDB_LITE
assert(!VerifySstFileChecksum(options, env_options, fname).ok());
Options options;
options.env = env;
EnvOptions env_options;
Status v = VerifySstFileChecksum(options, env_options, fname);
assert(!v.ok());
#endif
}
return s;
}
Status TruncateFile(Env* env, const std::string& fname, uint64_t new_length) {
uint64_t old_length;
Status s = env->GetFileSize(fname, &old_length);
if (!s.ok() || new_length == old_length) {
return s;
}
// Do it
std::string contents;
s = ReadFileToString(env, fname, &contents);
if (s.ok()) {
contents.resize(static_cast<size_t>(new_length), 'b');
s = WriteStringToFile(env, contents, fname);
}
return s;
}
} // namespace test
} // namespace ROCKSDB_NAMESPACE

7
test_util/testutil.h
View File

@ -22,9 +22,7 @@
#include "rocksdb/options.h"
#include "rocksdb/slice.h"
#include "rocksdb/table.h"
#include "table/block_based/block_based_table_factory.h"
#include "table/internal_iterator.h"
#include "table/plain/plain_table_factory.h"
#include "util/mutexlock.h"
namespace ROCKSDB_NAMESPACE {
@ -804,8 +802,9 @@ size_t GetLinesCount(const std::string& fname, const std::string& pattern);
// Tries to set TEST_TMPDIR to a directory supporting direct IO.
void ResetTmpDirForDirectIO();
void CorruptFile(const std::string& fname, int offset, int bytes_to_corrupt);
Status CorruptFile(Env* env, const std::string& fname, int offset,
int bytes_to_corrupt, bool verify_checksum = true);
Status TruncateFile(Env* env, const std::string& fname, uint64_t length);
} // namespace test
} // namespace ROCKSDB_NAMESPACE

1
util/gflags_compat.h
View File

@ -3,6 +3,7 @@
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#pragma once
#include <gflags/gflags.h>
#ifndef GFLAGS_NAMESPACE

19
utilities/persistent_cache/persistent_cache_test.cc
View File

@ -253,18 +253,19 @@ TEST_F(PersistentCacheTierTest, DISABLED_TieredCacheInsertWithEviction) {
}
std::shared_ptr<PersistentCacheTier> MakeVolatileCache(
const std::string& /*dbname*/) {
Env* /*env*/, const std::string& /*dbname*/) {
return std::make_shared<VolatileCacheTier>();
}
std::shared_ptr<PersistentCacheTier> MakeBlockCache(const std::string& dbname) {
return NewBlockCache(Env::Default(), dbname);
std::shared_ptr<PersistentCacheTier> MakeBlockCache(Env* env,
const std::string& dbname) {
return NewBlockCache(env, dbname);
}
std::shared_ptr<PersistentCacheTier> MakeTieredCache(
const std::string& dbname) {
Env* env, const std::string& dbname) {
const auto memory_size = 1 * 1024 * 1024 * kStressFactor;
return NewTieredCache(Env::Default(), dbname, static_cast<size_t>(memory_size));
return NewTieredCache(env, dbname, static_cast<size_t>(memory_size));
}
#ifdef OS_LINUX
@ -442,26 +443,26 @@ void PersistentCacheDBTest::RunTest(
// specifically written for Travis.
// Now used generally because main tests are too expensive as unit tests.
TEST_F(PersistentCacheDBTest, BasicTest) {
RunTest(std::bind(&MakeBlockCache, dbname_), /*max_keys=*/1024,
RunTest(std::bind(&MakeBlockCache, env_, dbname_), /*max_keys=*/1024,
/*max_usecase=*/1);
}
// test table with block page cache
// DISABLED for now (very expensive, especially memory)
TEST_F(PersistentCacheDBTest, DISABLED_BlockCacheTest) {
RunTest(std::bind(&MakeBlockCache, dbname_));
RunTest(std::bind(&MakeBlockCache, env_, dbname_));
}
// test table with volatile page cache
// DISABLED for now (very expensive, especially memory)
TEST_F(PersistentCacheDBTest, DISABLED_VolatileCacheTest) {
RunTest(std::bind(&MakeVolatileCache, dbname_));
RunTest(std::bind(&MakeVolatileCache, env_, dbname_));
}
// test table with tiered page cache
// DISABLED for now (very expensive, especially memory)
TEST_F(PersistentCacheDBTest, DISABLED_TieredCacheTest) {
RunTest(std::bind(&MakeTieredCache, dbname_));
RunTest(std::bind(&MakeTieredCache, env_, dbname_));
}
} // namespace ROCKSDB_NAMESPACE