From 960d936e8332546bd8a91cb0b0c3d06604368e83 Mon Sep 17 00:00:00 2001 From: Poornima Chozhiyath Raman Date: Wed, 5 Aug 2015 16:16:46 -0700 Subject: [PATCH] Add function 'GetInfoLogList()' Summary: The list of info log files of a db can be obtained using the new function. Test Plan: New test in db_test.cc passed. Reviewers: yhchiang, IslamAbdelRahman, sdong Reviewed By: sdong Subscribers: IslamAbdelRahman, leveldb, dhruba Differential Revision: https://reviews.facebook.net/D41715 --- db/db_test.cc | 56 +++++++++++++++++++++ include/rocksdb/utilities/info_log_finder.h | 19 +++++++ src.mk | 1 + utilities/convenience/info_log_finder.cc | 48 ++++++++++++++++++ 4 files changed, 124 insertions(+) create mode 100644 include/rocksdb/utilities/info_log_finder.h create mode 100644 utilities/convenience/info_log_finder.cc diff --git a/db/db_test.cc b/db/db_test.cc index db8f03795..52f800eeb 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -48,6 +48,7 @@ #include "rocksdb/thread_status.h" #include "rocksdb/utilities/write_batch_with_index.h" #include "rocksdb/utilities/checkpoint.h" +#include "rocksdb/utilities/info_log_finder.h" #include "rocksdb/utilities/optimistic_transaction_db.h" #include "table/block_based_table_factory.h" #include "table/mock_table.h" @@ -3847,9 +3848,19 @@ class WrappedBloom : public FilterPolicy { }; } // namespace +<<<<<<< HEAD TEST_F(DBTest, BloomFilterWrapper) { Options options = CurrentOptions(); options.statistics = rocksdb::CreateDBStatistics(); +======= +// Test scope: +// - We expect to open the data store when there is incomplete trailing writes +// at the end of any of the logs +// - We do not expect to open the data store for corruption +TEST_F(DBTest, kTolerateCorruptedTailRecords) { + const int jstart = RecoveryTestHelper::kWALFileOffset; + const int jend = jstart + RecoveryTestHelper::kWALFilesCount; +>>>>>>> Info Log List can be obtained BlockBasedTableOptions table_options; WrappedBloom* policy = new WrappedBloom(10); @@ -3867,12 +3878,21 @@ TEST_F(DBTest, BloomFilterWrapper) { ASSERT_EQ(0U, policy->GetCounter()); Flush(1); +<<<<<<< HEAD // Check if they can be found for (int i = 0; i < maxKey; i++) { ASSERT_EQ(Key(i), Get(1, Key(i))); } ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0); ASSERT_EQ(1U * maxKey, policy->GetCounter()); +======= +// Test scope: +// We don't expect the data store to be opened if there is any corruption +// (leading, middle or trailing -- incomplete writes or corruption) +TEST_F(DBTest, kAbsoluteConsistency) { + const int jstart = RecoveryTestHelper::kWALFileOffset; + const int jend = jstart + RecoveryTestHelper::kWALFilesCount; +>>>>>>> Info Log List can be obtained // Check if filter is useful for (int i = 0; i < maxKey; i++) { @@ -3941,6 +3961,18 @@ TEST_F(DBTest, SnapshotFiles) { } CopyFile(src, dest, size); } +<<<<<<< HEAD +======= + } +} + +// Test scope: +// - We expect to open the data store under all scenarios +// - We expect to have recovered records past the corruption zone +TEST_F(DBTest, kSkipAnyCorruptedRecords) { + const int jstart = RecoveryTestHelper::kWALFileOffset; + const int jend = jstart + RecoveryTestHelper::kWALFilesCount; +>>>>>>> Info Log List can be obtained // release file snapshot dbfull()->DisableFileDeletions(); @@ -5912,6 +5944,7 @@ TEST_F(DBTest, DBIteratorBoundTest) { } } +<<<<<<< HEAD TEST_F(DBTest, WriteSingleThreadEntry) { std::vector threads; dbfull()->TEST_LockMutex(); @@ -5928,6 +5961,29 @@ TEST_F(DBTest, WriteSingleThreadEntry) { for (auto& t : threads) { t.join(); } +======= +TEST_F(DBTest, InfoLogFileList) { + Options options = CurrentOptions(); + options.db_log_dir = test::TmpDir(env_); + options.max_background_flushes = 0; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "key", "value")); + std::vector result; + auto s = GetInfoLogList(db_, &result); + ASSERT_TRUE(s.ok()); + ASSERT_GT(result.size(), 0); +} + +TEST_F(DBTest, PreShutdownFlush) { + Options options = CurrentOptions(); + options.max_background_flushes = 0; + CreateAndReopenWithCF({"pikachu"}, options); + ASSERT_OK(Put(1, "key", "value")); + CancelAllBackgroundWork(db_); + Status s = + db_->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr); + ASSERT_TRUE(s.IsShutdownInProgress()); +>>>>>>> Info Log List can be obtained } TEST_F(DBTest, DisableDataSyncTest) { diff --git a/include/rocksdb/utilities/info_log_finder.h b/include/rocksdb/utilities/info_log_finder.h new file mode 100644 index 000000000..916c54c28 --- /dev/null +++ b/include/rocksdb/utilities/info_log_finder.h @@ -0,0 +1,19 @@ +// Copyright (c) 2014, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once + +#include +#include + +#include "rocksdb/db.h" +#include "rocksdb/options.h" + +namespace rocksdb { + +// This function can be used to list the Information logs, +// given the db pointer. +Status GetInfoLogList(DB* db, std::vector* info_log_list); +} // namespace rocksdb diff --git a/src.mk b/src.mk index f35d81954..860d7ab68 100644 --- a/src.mk +++ b/src.mk @@ -99,6 +99,7 @@ LIB_SOURCES = \ util/instrumented_mutex.cc \ util/iostats_context.cc \ utilities/backupable/backupable_db.cc \ + utilities/convenience/info_log_finder.cc \ utilities/checkpoint/checkpoint.cc \ utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc \ utilities/document/document_db.cc \ diff --git a/utilities/convenience/info_log_finder.cc b/utilities/convenience/info_log_finder.cc new file mode 100644 index 000000000..acdec5119 --- /dev/null +++ b/utilities/convenience/info_log_finder.cc @@ -0,0 +1,48 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2012 Facebook. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "db/filename.h" +#include "rocksdb/env.h" +#include "rocksdb/utilities/info_log_finder.h" + +namespace rocksdb { + +Status GetInfoLogList(DB* db, std::vector* info_log_list) { + uint64_t number = 0; + FileType type; + std::string path; + + if (!db) { + return Status::InvalidArgument("DB pointer is not valid"); + } + + const Options& options = db->GetOptions(); + if (!options.db_log_dir.empty()) { + path = options.db_log_dir; + } else { + path = db->GetName(); + } + InfoLogPrefix info_log_prefix(!options.db_log_dir.empty(), db->GetName()); + auto* env = options.env; + std::vector file_names; + Status s = env->GetChildren(path, &file_names); + + if (!s.ok()) { + return s; + } + + for (auto f : file_names) { + if (ParseFileName(f, &number, info_log_prefix.prefix, &type) && + (type == kInfoLogFile)) { + info_log_list->push_back(f); + } + } + return Status::OK(); +} +} // namespace rocksdb