andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make FileType Public and Replace kLogFile with kWalFile (#7580)

Browse Source 
Summary:
As suggested by pdillinger ,The name of kLogFile is misleading, in some tests, kLogFile is defined as info log. Replace it with kWalFile and move it to public, which will be used in https://github.com/facebook/rocksdb/issues/7523

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7580

Test Plan: make check

Reviewed By: riversand963

Differential Revision: D24485420

Pulled By: zhichao-cao

fbshipit-source-id: 955e3dacc1021bb590fde93b0a568ffe9ad80799
This commit is contained in:
Zhichao Cao 2020-10-22 17:04:39 -07:00 committed by Facebook GitHub Bot
parent 1c78e4b235
commit d8ec0a760a
25 changed files with 80 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db/column_family_test.cc
View File

@ -3271,7 +3271,7 @@ TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) {
FileType type;
if (!(ParseFileName(filenames[i], &number, &type))) continue;
if (type != kLogFile) continue;
if (type != kWalFile) continue;
logfs.push_back(filenames[i]);
}

4
db/corruption_test.cc
View File

@ -249,8 +249,8 @@ TEST_F(CorruptionTest, Recovery) {
// is not available for WAL though.
CloseDb();
#endif
Corrupt(kLogFile, 19, 1); // WriteBatch tag for first record
Corrupt(kLogFile, log::kBlockSize + 1000, 1); // Somewhere in second block
Corrupt(kWalFile, 19, 1); // WriteBatch tag for first record
Corrupt(kWalFile, log::kBlockSize + 1000, 1); // Somewhere in second block
ASSERT_TRUE(!TryReopen().ok());
options_.paranoid_checks = false;
Reopen(&options_);

2
db/db_basic_test.cc
View File

@ -2405,7 +2405,7 @@ TEST_F(DBBasicTest, RecoverWithNoManifest) {
ASSERT_OK(env_->GetChildren(dbname_, &files));
for (const auto& file : files) {
uint64_t number = 0;
FileType type = kLogFile;
FileType type = kWalFile;
if (ParseFileName(file, &number, &type) && type == kDescriptorFile) {
ASSERT_OK(env_->DeleteFile(dbname_ + "/" + file));
}

10
db/db_impl/db_impl.cc
View File

@ -3430,14 +3430,14 @@ Status DBImpl::DeleteFile(std::string name) {
FileType type;
WalFileType log_type;
if (!ParseFileName(name, &number, &type, &log_type) ||
(type != kTableFile && type != kLogFile)) {
(type != kTableFile && type != kWalFile)) {
ROCKS_LOG_ERROR(immutable_db_options_.info_log, "DeleteFile %s failed.\n",
name.c_str());
return Status::InvalidArgument("Invalid file name");
}
Status status;
if (type == kLogFile) {
if (type == kWalFile) {
// Only allow deleting archived log files
if (log_type != kArchivedLogFile) {
ROCKS_LOG_ERROR(immutable_db_options_.info_log,
@ -3862,7 +3862,7 @@ Status DestroyDB(const std::string& dbname, const Options& options,
std::string path_to_delete = dbname + "/" + fname;
if (type == kMetaDatabase) {
del = DestroyDB(path_to_delete, options);
} else if (type == kTableFile || type == kLogFile) {
} else if (type == kTableFile || type == kWalFile) {
del = DeleteDBFile(&soptions, path_to_delete, dbname,
/*force_bg=*/false, /*force_fg=*/!wal_in_db_path);
} else {
@ -3916,7 +3916,7 @@ Status DestroyDB(const std::string& dbname, const Options& options,
if (env->GetChildren(archivedir, &archiveFiles).ok()) {
// Delete archival files.
for (const auto& file : archiveFiles) {
if (ParseFileName(file, &number, &type) && type == kLogFile) {
if (ParseFileName(file, &number, &type) && type == kWalFile) {
Status del =
DeleteDBFile(&soptions, archivedir + "/" + file, archivedir,
/*force_bg=*/false, /*force_fg=*/!wal_in_db_path);
@ -3932,7 +3932,7 @@ Status DestroyDB(const std::string& dbname, const Options& options,
// Delete log files in the WAL dir
if (wal_dir_exists) {
for (const auto& file : walDirFiles) {
if (ParseFileName(file, &number, &type) && type == kLogFile) {
if (ParseFileName(file, &number, &type) && type == kWalFile) {
Status del =
DeleteDBFile(&soptions, LogFileName(soptions.wal_dir, number),
soptions.wal_dir, /*force_bg=*/false,

8
db/db_impl/db_impl_files.cc
View File

@ -319,7 +319,7 @@ void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
const_cast<std::string*>(&fname));
Status file_deletion_status;
if (type == kTableFile || type == kBlobFile || type == kLogFile) {
if (type == kTableFile || type == kBlobFile || type == kWalFile) {
file_deletion_status =
DeleteDBFile(&immutable_db_options_, fname, path_to_sync,
/*force_bg=*/false, /*force_fg=*/!wal_in_db_path_);
@ -466,7 +466,7 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
bool keep = true;
switch (type) {
case kLogFile:
case kWalFile:
keep = ((number >= state.log_number) ||
(number == state.prev_log_number) ||
(log_recycle_files_set.find(number) !=
@ -546,7 +546,7 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
dir_to_sync = candidate_file.file_path;
} else {
dir_to_sync =
(type == kLogFile) ? immutable_db_options_.wal_dir : dbname_;
(type == kWalFile) ? immutable_db_options_.wal_dir : dbname_;
fname = dir_to_sync +
((!dir_to_sync.empty() && dir_to_sync.back() == '/') ||
(!to_delete.empty() && to_delete.front() == '/')
@ -556,7 +556,7 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
}
#ifndef ROCKSDB_LITE
if (type == kLogFile && (immutable_db_options_.wal_ttl_seconds > 0 ||
if (type == kWalFile && (immutable_db_options_.wal_ttl_seconds > 0 ||
immutable_db_options_.wal_size_limit_mb > 0)) {
wal_manager_.ArchiveWALFile(fname, number);
continue;

4
db/db_impl/db_impl_open.cc
View File

@ -401,7 +401,7 @@ Status DBImpl::Recover(
}
for (const std::string& file : files_in_dbname) {
uint64_t number = 0;
FileType type = kLogFile; // initialize
FileType type = kWalFile; // initialize
if (ParseFileName(file, &number, &type) && type == kDescriptorFile) {
// Found MANIFEST (descriptor log), thus best-efforts recovery does
// not have to treat the db as empty.
@ -575,7 +575,7 @@ Status DBImpl::Recover(
for (const auto& file : files_in_wal_dir) {
uint64_t number;
FileType type;
if (ParseFileName(file, &number, &type) && type == kLogFile) {
if (ParseFileName(file, &number, &type) && type == kWalFile) {
if (is_new_db) {
return Status::Corruption(
"While creating a new Db, wal_dir contains "

2
db/db_impl/db_impl_secondary.cc
View File

@ -112,7 +112,7 @@ Status DBImplSecondary::FindNewLogNumbers(std::vector<uint64_t>* logs) {
for (size_t i = 0; i < filenames.size(); i++) {
uint64_t number;
FileType type;
if (ParseFileName(filenames[i], &number, &type) && type == kLogFile &&
if (ParseFileName(filenames[i], &number, &type) && type == kWalFile &&
number >= log_number_min) {
logs->push_back(number);
}

2
db/db_impl/db_secondary_test.cc
View File

@ -104,7 +104,7 @@ void DBSecondaryTest::CheckFileTypeCounts(const std::string& dir,
uint64_t number;
FileType type;
if (ParseFileName(file, &number, &type)) {
log_cnt += (type == kLogFile);
log_cnt += (type == kWalFile);
sst_cnt += (type == kTableFile);
manifest_cnt += (type == kDescriptorFile);
}

4
db/db_info_dumper.cc
View File

@ -62,7 +62,7 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
dbname.c_str(), file.c_str());
}
break;
case kLogFile:
case kWalFile:
if (env->GetFileSize(dbname + "/" + file, &file_size).ok()) {
char str[16];
snprintf(str, sizeof(str), "%" PRIu64, file_size);
@ -118,7 +118,7 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
wal_info.clear();
for (const std::string& file : files) {
if (ParseFileName(file, &number, &type)) {
if (type == kLogFile) {
if (type == kWalFile) {
if (env->GetFileSize(options.wal_dir + "/" + file, &file_size).ok()) {
char str[16];
snprintf(str, sizeof(str), "%" PRIu64, file_size);

2
db/db_test2.cc
View File

@ -4569,7 +4569,7 @@ TEST_F(DBTest2, CrashInRecoveryMultipleCF) {
for (const auto& f : filenames) {
uint64_t number;
FileType type;
if (ParseFileName(f, &number, &type) && type == FileType::kLogFile) {
if (ParseFileName(f, &number, &type) && type == FileType::kWalFile) {
std::string fname = dbname_ + "/" + f;
std::string file_content;
ASSERT_OK(ReadFileToString(env_, fname, &file_content));

2
db/deletefile_test.cc
View File

@ -112,7 +112,7 @@ class DeleteFileTest : public DBTestBase {
uint64_t number;
FileType type;
if (ParseFileName(file, &number, &type)) {
log_cnt += (type == kLogFile);
log_cnt += (type == kWalFile);
sst_cnt += (type == kTableFile);
manifest_cnt += (type == kDescriptorFile);
}

36
db/filename_test.cc
View File

@ -35,23 +35,23 @@ TEST_F(FileNameTest, Parse) {
FileType type;
char mode;
} cases[] = {
{"100.log", 100, kLogFile, kAllMode},
{"0.log", 0, kLogFile, kAllMode},
{"0.sst", 0, kTableFile, kAllMode},
{"CURRENT", 0, kCurrentFile, kAllMode},
{"LOCK", 0, kDBLockFile, kAllMode},
{"MANIFEST-2", 2, kDescriptorFile, kAllMode},
{"MANIFEST-7", 7, kDescriptorFile, kAllMode},
{"METADB-2", 2, kMetaDatabase, kAllMode},
{"METADB-7", 7, kMetaDatabase, kAllMode},
{"LOG", 0, kInfoLogFile, kDefautInfoLogDir},
{"LOG.old", 0, kInfoLogFile, kDefautInfoLogDir},
{"LOG.old.6688", 6688, kInfoLogFile, kDefautInfoLogDir},
{"rocksdb_dir_LOG", 0, kInfoLogFile, kDifferentInfoLogDir},
{"rocksdb_dir_LOG.old", 0, kInfoLogFile, kDifferentInfoLogDir},
{"rocksdb_dir_LOG.old.6688", 6688, kInfoLogFile, kDifferentInfoLogDir},
{"18446744073709551615.log", 18446744073709551615ull, kLogFile,
kAllMode}, };
{"100.log", 100, kWalFile, kAllMode},
{"0.log", 0, kWalFile, kAllMode},
{"0.sst", 0, kTableFile, kAllMode},
{"CURRENT", 0, kCurrentFile, kAllMode},
{"LOCK", 0, kDBLockFile, kAllMode},
{"MANIFEST-2", 2, kDescriptorFile, kAllMode},
{"MANIFEST-7", 7, kDescriptorFile, kAllMode},
{"METADB-2", 2, kMetaDatabase, kAllMode},
{"METADB-7", 7, kMetaDatabase, kAllMode},
{"LOG", 0, kInfoLogFile, kDefautInfoLogDir},
{"LOG.old", 0, kInfoLogFile, kDefautInfoLogDir},
{"LOG.old.6688", 6688, kInfoLogFile, kDefautInfoLogDir},
{"rocksdb_dir_LOG", 0, kInfoLogFile, kDifferentInfoLogDir},
{"rocksdb_dir_LOG.old", 0, kInfoLogFile, kDifferentInfoLogDir},
{"rocksdb_dir_LOG.old.6688", 6688, kInfoLogFile, kDifferentInfoLogDir},
{"18446744073709551615.log", 18446744073709551615ull, kWalFile, kAllMode},
};
for (char mode : {kDifferentInfoLogDir, kDefautInfoLogDir, kNoCheckLogDir}) {
for (unsigned int i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
InfoLogPrefix info_log_prefix(mode != kDefautInfoLogDir, "/rocksdb/dir");
@ -142,7 +142,7 @@ TEST_F(FileNameTest, Construction) {
ASSERT_EQ("foo/", std::string(fname.data(), 4));
ASSERT_TRUE(ParseFileName(fname.c_str() + 4, &number, &type));
ASSERT_EQ(192U, number);
ASSERT_EQ(kLogFile, type);
ASSERT_EQ(kWalFile, type);
fname = TableFileName({DbPath("bar", 0)}, 200, 0);
std::string fname1 =

2
db/obsolete_files_test.cc
View File

@ -74,7 +74,7 @@ class ObsoleteFilesTest : public DBTestBase {
uint64_t number;
FileType type;
if (ParseFileName(file, &number, &type)) {
log_cnt += (type == kLogFile);
log_cnt += (type == kWalFile);
sst_cnt += (type == kTableFile);
manifest_cnt += (type == kDescriptorFile);
}

2
db/repair.cc
View File

@ -312,7 +312,7 @@ class Repairer {
if (number + 1 > next_file_number_) {
next_file_number_ = number + 1;
}
if (type == kLogFile) {
if (type == kWalFile) {
logs_.push_back(number);
} else if (type == kTableFile) {
table_fds_.emplace_back(number, static_cast<uint32_t>(path_id),

4
db/wal_manager.cc
View File

@ -175,7 +175,7 @@ void WalManager::PurgeObsoleteWALFiles() {
for (auto& f : files) {
uint64_t number;
FileType type;
if (ParseFileName(f, &number, &type) && type == kLogFile) {
if (ParseFileName(f, &number, &type) && type == kWalFile) {
std::string const file_path = archival_dir + "/" + f;
if (ttl_enabled) {
uint64_t file_m_time;
@ -292,7 +292,7 @@ Status WalManager::GetSortedWalsOfType(const std::string& path,
for (const auto& f : all_files) {
uint64_t number;
FileType type;
if (ParseFileName(f, &number, &type) && type == kLogFile) {
if (ParseFileName(f, &number, &type) && type == kWalFile) {
SequenceNumber sequence;
Status s = ReadFirstRecord(log_type, number, &sequence);
if (!s.ok()) {

10
db/wal_manager_test.cc
View File

@ -171,7 +171,7 @@ uint64_t GetLogDirSize(std::string dir_path, Env* env) {
for (auto& f : files) {
uint64_t number;
FileType type;
if (ParseFileName(f, &number, &type) && type == kLogFile) {
if (ParseFileName(f, &number, &type) && type == kWalFile) {
std::string const file_path = dir_path + "/" + f;
uint64_t file_size;
env->GetFileSize(file_path, &file_size);
@ -232,7 +232,7 @@ TEST_F(WalManagerTest, WALArchivalSizeLimit) {
CreateArchiveLogs(20, 5000);
std::vector<std::uint64_t> log_files =
ListSpecificFiles(env_.get(), archive_dir, kLogFile);
ListSpecificFiles(env_.get(), archive_dir, kWalFile);
ASSERT_EQ(log_files.size(), 20U);
db_options_.wal_size_limit_mb = 8;
@ -247,7 +247,7 @@ TEST_F(WalManagerTest, WALArchivalSizeLimit) {
Reopen();
wal_manager_->PurgeObsoleteWALFiles();
log_files = ListSpecificFiles(env_.get(), archive_dir, kLogFile);
log_files = ListSpecificFiles(env_.get(), archive_dir, kWalFile);
ASSERT_TRUE(log_files.empty());
}
@ -265,7 +265,7 @@ TEST_F(WalManagerTest, WALArchivalTtl) {
CreateArchiveLogs(20, 5000);
std::vector<uint64_t> log_files =
ListSpecificFiles(env_.get(), archive_dir, kLogFile);
ListSpecificFiles(env_.get(), archive_dir, kWalFile);
ASSERT_GT(log_files.size(), 0U);
db_options_.wal_ttl_seconds = 1;
@ -273,7 +273,7 @@ TEST_F(WalManagerTest, WALArchivalTtl) {
Reopen();
wal_manager_->PurgeObsoleteWALFiles();
log_files = ListSpecificFiles(env_.get(), archive_dir, kLogFile);
log_files = ListSpecificFiles(env_.get(), archive_dir, kWalFile);
ASSERT_TRUE(log_files.empty());
}

4
file/filename.cc
View File

@ -352,7 +352,7 @@ bool ParseFileName(const std::string& fname, uint64_t* number,
Slice suffix = rest;
if (suffix == Slice("log")) {
*type = kLogFile;
*type = kWalFile;
if (log_type && !archive_dir_found) {
*log_type = kAliveLogFile;
}
@ -432,7 +432,7 @@ Status GetInfoLogFiles(Env* env, const std::string& db_log_dir,
assert(parent_dir != nullptr);
assert(info_log_list != nullptr);
uint64_t number = 0;
FileType type = kLogFile;
FileType type = kWalFile;
if (!db_log_dir.empty()) {
*parent_dir = db_log_dir;

14
file/filename.h
View File

@ -35,20 +35,6 @@ const char kFilePathSeparator = '\\';
const char kFilePathSeparator = '/';
#endif
enum FileType {
kLogFile,
kDBLockFile,
kTableFile,
kDescriptorFile,
kCurrentFile,
kTempFile,
kInfoLogFile, // Either the current one, or an old one
kMetaDatabase,
kIdentityFile,
kOptionsFile,
kBlobFile
};
// Return the name of the log file with the specified number
// in the db named by "dbname". The result will be prefixed with
// "dbname".

1
include/rocksdb/options.h
View File

@ -24,6 +24,7 @@
#include "rocksdb/file_checksum.h"
#include "rocksdb/listener.h"
#include "rocksdb/sst_partitioner.h"
#include "rocksdb/types.h"
#include "rocksdb/universal_compaction.h"
#include "rocksdb/version.h"
#include "rocksdb/write_buffer_manager.h"

16
include/rocksdb/types.h
View File

@ -19,6 +19,22 @@ typedef uint64_t SequenceNumber;
const SequenceNumber kMinUnCommittedSeq = 1; // 0 is always committed
// The types of files RocksDB uses in a DB directory. (Available for
// advanced options.)
enum FileType {
kWalFile,
kDBLockFile,
kTableFile,
kDescriptorFile,
kCurrentFile,
kTempFile,
kInfoLogFile, // Either the current one, or an old one
kMetaDatabase,
kIdentityFile,
kOptionsFile,
kBlobFile
};
// User-oriented representation of internal key types.
// Ordering of this enum entries should not change.
enum EntryType {

4
tools/ldb_cmd.cc
View File

@ -1129,7 +1129,7 @@ void ManifestDumpCommand::DoCommand() {
fname = file_path;
}
uint64_t file_num = 0;
FileType file_type = kLogFile; // Just for initialization
FileType file_type = kWalFile; // Just for initialization
if (ParseFileName(fname, &file_num, &file_type) &&
file_type == kDescriptorFile) {
if (!matched_file.empty()) {
@ -1672,7 +1672,7 @@ void DBDumperCommand::DoCommand() {
}
switch (type) {
case kLogFile:
case kWalFile:
// TODO(myabandeh): allow configuring is_write_commited
DumpWalFile(options_, path_, /* print_header_ */ true,
/* print_values_ */ true, true /* is_write_commited */,

13
utilities/backupable/backupable_db.cc
View File

@ -1013,7 +1013,7 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
uint64_t size_limit_bytes, FileType type,
const std::string& checksum_func_name,
const std::string& checksum_val) {
if (type == kLogFile && !options_.backup_log_files) {
if (type == kWalFile && !options_.backup_log_files) {
return Status::OK();
}
Log(options_.info_log, "add file for backup %s", fname.c_str());
@ -1024,7 +1024,7 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
}
EnvOptions src_env_options;
switch (type) {
case kLogFile:
case kWalFile:
src_env_options =
db_env_->OptimizeForLogRead(src_raw_env_options);
break;
@ -1315,7 +1315,7 @@ Status BackupEngineImpl::RestoreDBFromBackup(const RestoreOptions& options,
if (options.keep_log_files) {
// delete files in db_dir, but keep all the log files
DeleteChildren(db_dir, 1 << kLogFile);
DeleteChildren(db_dir, 1 << kWalFile);
// move all the files from archive dir to wal_dir
std::string archive_dir = ArchivalDirectory(wal_dir);
std::vector<std::string> archive_files;
@ -1324,7 +1324,7 @@ Status BackupEngineImpl::RestoreDBFromBackup(const RestoreOptions& options,
uint64_t number;
FileType type;
bool ok = ParseFileName(f, &number, &type);
if (ok && type == kLogFile) {
if (ok && type == kWalFile) {
ROCKS_LOG_INFO(options_.info_log,
"Moving log file from archive/ to wal_dir: %s",
f.c_str());
@ -1377,9 +1377,8 @@ Status BackupEngineImpl::RestoreDBFromBackup(const RestoreOptions& options,
dst);
}
// 3. Construct the final path
// kLogFile lives in wal_dir and all the rest live in db_dir
dst = ((type == kLogFile) ? wal_dir : db_dir) +
"/" + dst;
// kWalFile lives in wal_dir and all the rest live in db_dir
dst = ((type == kWalFile) ? wal_dir : db_dir) + "/" + dst;
ROCKS_LOG_INFO(options_.info_log, "Restoring %s to %s\n", file.c_str(),
dst.c_str());

2
utilities/backupable/backupable_db_test.cc
View File

@ -743,7 +743,7 @@ class BackupableDBTest : public testing::Test {
uint64_t number;
FileType type;
bool ok = ParseFileName(f, &number, &type);
if (ok && type == kLogFile) {
if (ok && type == kWalFile) {
db_chroot_env_->DeleteFile(dbname_ + "/" + f);
}
}

6
utilities/checkpoint/checkpoint_impl.cc
View File

@ -373,14 +373,14 @@ Status CheckpointImpl::CreateCustomCheckpoint(
live_wal_files[i]->LogNumber() >= min_log_num)) {
if (i + 1 == wal_size) {
s = copy_file_cb(db_options.wal_dir, live_wal_files[i]->PathName(),
live_wal_files[i]->SizeFileBytes(), kLogFile,
live_wal_files[i]->SizeFileBytes(), kWalFile,
kUnknownFileChecksumFuncName, kUnknownFileChecksum);
break;
}
if (same_fs) {
// we only care about live log files
s = link_file_cb(db_options.wal_dir, live_wal_files[i]->PathName(),
kLogFile);
kWalFile);
if (s.IsNotSupported()) {
same_fs = false;
s = Status::OK();
@ -388,7 +388,7 @@ Status CheckpointImpl::CreateCustomCheckpoint(
}
if (!same_fs) {
s = copy_file_cb(db_options.wal_dir, live_wal_files[i]->PathName(), 0,
kLogFile, kUnknownFileChecksumFuncName,
kWalFile, kUnknownFileChecksumFuncName,
kUnknownFileChecksum);
}
}

2
utilities/checkpoint/checkpoint_test.cc
View File

@ -668,7 +668,7 @@ TEST_F(CheckpointTest, CurrentFileModifiedWhileCheckpointing2PC) {
uint64_t num;
FileType type;
WalFileType log_type;
if (ParseFileName(file, &num, &type, &log_type) && type == kLogFile) {
if (ParseFileName(file, &num, &type, &log_type) && type == kWalFile) {
num_log_files++;
}
}