andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support options.max_open_files != -1 with FIFO compaction (#6503)

Browse Source 
Summary:
Allow user to specify options.max_open_files != -1 with FIFO compaction.
If max_open_files != -1, not all table files are kept open.
In the past, FIFO style compaction requires all table files to be open in order
to read file creation time from table properties. Later, we added file creation
time to MANIFEST, making it possible to read file creation time without opening
file.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6503

Test Plan: make check

Differential Revision: D20353758

Pulled By: riversand963

fbshipit-source-id: ba5c61a648419e47e9ef6d74e0e280e3ee24f296
This commit is contained in:
Yanqin Jin 2020-03-09 18:42:27 -07:00 committed by Facebook Github Bot
parent d93812c9ae
commit fd1da22111
3 changed files with 33 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
db/compaction/compaction_picker_fifo.cc
View File

@ -70,12 +70,9 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
// avoid underflow
if (current_time > mutable_cf_options.ttl) {
for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
auto f = *ritr;
if (f->fd.table_reader != nullptr &&
f->fd.table_reader->GetTableProperties() != nullptr) {
auto creation_time =
f->fd.table_reader->GetTableProperties()->creation_time;
if (creation_time == 0 ||
FileMetaData* f = *ritr;
uint64_t creation_time = f->TryGetFileCreationTime();
if (creation_time == kUnknownFileCreationTime ||
creation_time >= (current_time - mutable_cf_options.ttl)) {
break;
}
@ -83,7 +80,6 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
inputs[0].files.push_back(f);
}
}
}
// Return a nullptr and proceed to size-based FIFO compaction if:
// 1. there are no files older than ttl OR
@ -100,7 +96,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
"[%s] FIFO compaction: picking file %" PRIu64
" with creation time %" PRIu64 " for deletion",
cf_name.c_str(), f->fd.GetNumber(),
f->fd.table_reader->GetTableProperties()->creation_time);
f->TryGetFileCreationTime());
}
Compaction* c = new Compaction(

25
db/db_compaction_test.cc
View File

@ -5151,6 +5151,31 @@ TEST_P(DBCompactionTestWithParam,
}
}
TEST_F(DBCompactionTest, FifoCompactionGetFileCreationTime) {
MockEnv mock_env(env_);
do {
Options options = CurrentOptions();
options.table_factory.reset(new BlockBasedTableFactory());
options.env = &mock_env;
options.ttl = static_cast<uint64_t>(24) * 3600;
options.compaction_style = kCompactionStyleFIFO;
constexpr size_t kNumFiles = 24;
options.max_open_files = 20;
constexpr size_t kNumKeysPerFile = 10;
DestroyAndReopen(options);
for (size_t i = 0; i < kNumFiles; ++i) {
for (size_t j = 0; j < kNumKeysPerFile; ++j) {
ASSERT_OK(Put(std::to_string(j), "value_" + std::to_string(i)));
}
ASSERT_OK(Flush());
}
mock_env.FakeSleepForMicroseconds(
static_cast<uint64_t>(1000 * 1000 * (1 + options.ttl)));
ASSERT_OK(Put("foo", "value"));
ASSERT_OK(Flush());
} while (ChangeOptions());
}
#endif // !defined(ROCKSDB_LITE)
} // namespace ROCKSDB_NAMESPACE

2
include/rocksdb/advanced_options.h
View File

@ -645,7 +645,6 @@ struct AdvancedColumnFamilyOptions {
bool report_bg_io_stats = false;
// Files older than TTL will go through the compaction process.
// Pre-req: This needs max_open_files to be set to -1.
// In Level: Non-bottom-level files older than TTL will go through the
// compation process.
// In FIFO: Files older than TTL will be deleted.
@ -673,7 +672,6 @@ struct AdvancedColumnFamilyOptions {
// Supported in Level and FIFO compaction.
// In FIFO compaction, this option has the same meaning as TTL and whichever
// stricter will be used.
// Pre-req: max_open_file == -1.
// unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60
//
// Values: