andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Track per-SST user-defined timestamp information in MANIFEST (#9092)

Browse Source 
Summary:
Track per-SST user-defined timestamp information in MANIFEST https://github.com/facebook/rocksdb/issues/8957

Rockdb has supported user-defined timestamp feature. Application can specify a timestamp
when writing each k-v pair. When data flush from memory to disk file called SST files, file
creation activity will commit to MANIFEST. This commit is for tracking timestamp info in the
MANIFEST for each file. The changes involved are as follows:
1) Track max/min timestamp in FileMetaData, and fix invoved codes.
2) Add NewFileCustomTag::kMinTimestamp and NewFileCustomTag::kMinTimestamp in
    NewFileCustomTag ( in the kNewFile4 part ), and support invoved codes such as
    VersionEdit Encode and Decode etc.
3) Add unit test code for VersionEdit EncodeDecodeNewFile4, and fix invoved test codes.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9092

Reviewed By: ajkr, akankshamahajan15

Differential Revision: D32252323

Pulled By: riversand963

fbshipit-source-id: d2642898d6e3ad1fef0eb866b98045408bd4e162
This commit is contained in:
slk 2021-11-10 10:47:53 -08:00 committed by Facebook GitHub Bot
parent 1a8eec461b
commit 937fbcbddc
18 changed files with 182 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -3,6 +3,7 @@
### New Features
* Added new ChecksumType kXXH3 which is faster than kCRC32c on almost all x86\_64 hardware.
* Added a new online consistency check for BlobDB which validates that the number/total size of garbage blobs does not exceed the number/total size of all blobs in any given blob file.
* Provided support for tracking per-sst user-defined timestamp information in MANIFEST.
### Bug Fixes
* Prevent a `CompactRange()` with `CompactRangeOptions::change_level == true` from possibly causing corruption to the LSM state (overlapping files within a level) when run in parallel with another manual compaction. Note that setting `force_consistency_checks == true` (the default) would cause the DB to enter read-only mode in this scenario and return `Status::Corruption`, rather than committing any corruption.

3
db/compaction/compaction_job_test.cc
View File

@ -205,7 +205,8 @@ class CompactionJobTestBase : public testing::Test {
edit.AddFile(level, file_number, 0, 10, smallest_key, largest_key,
smallest_seqno, largest_seqno, false, oldest_blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
mutex_.Lock();
EXPECT_OK(

3
db/compaction/compaction_picker_test.cc
View File

@ -114,7 +114,8 @@ class CompactionPickerTest : public testing::Test {
InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
largest_seq, marked_for_compact, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
f->compensated_file_size =
(compensated_file_size != 0) ? compensated_file_size : file_size;
f->temperature = temperature;

20
db/db_impl/db_impl_compaction_flush.cc
View File

@ -1589,12 +1589,12 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
edit.SetColumnFamily(cfd->GetID());
for (const auto& f : vstorage->LevelFiles(level)) {
edit.DeleteFile(level, f->fd.GetNumber());
edit.AddFile(to_level, f->fd.GetNumber(), f->fd.GetPathId(),
f->fd.GetFileSize(), f->smallest, f->largest,
f->fd.smallest_seqno, f->fd.largest_seqno,
edit.AddFile(
to_level, f->fd.GetNumber(), f->fd.GetPathId(), f->fd.GetFileSize(),
f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->oldest_blob_file_number,
f->oldest_ancester_time, f->file_creation_time,
f->file_checksum, f->file_checksum_func_name);
f->oldest_ancester_time, f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->min_timestamp, f->max_timestamp);
}
ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
"[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@ -3170,13 +3170,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
for (size_t i = 0; i < c->num_input_files(l); i++) {
FileMetaData* f = c->input(l, i);
c->edit()->DeleteFile(c->level(l), f->fd.GetNumber());
c->edit()->AddFile(c->output_level(), f->fd.GetNumber(),
f->fd.GetPathId(), f->fd.GetFileSize(), f->smallest,
f->largest, f->fd.smallest_seqno,
c->edit()->AddFile(
c->output_level(), f->fd.GetNumber(), f->fd.GetPathId(),
f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
f->fd.largest_seqno, f->marked_for_compaction,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum,
f->file_checksum_func_name);
f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp);
ROCKS_LOG_BUFFER(
log_buffer,

3
db/db_impl/db_impl_experimental.cc
View File

@ -131,7 +131,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->oldest_blob_file_number,
f->oldest_ancester_time, f->file_creation_time,
f->file_checksum, f->file_checksum_func_name);
f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp);
}
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),

3
db/db_impl/db_impl_open.cc
View File

@ -1463,7 +1463,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
meta.fd.smallest_seqno, meta.fd.largest_seqno,
meta.marked_for_compaction, meta.oldest_blob_file_number,
meta.oldest_ancester_time, meta.file_creation_time,
meta.file_checksum, meta.file_checksum_func_name);
meta.file_checksum, meta.file_checksum_func_name,
meta.min_timestamp, meta.max_timestamp);
for (const auto& blob : blob_file_additions) {
edit->AddBlobFile(blob);

1
db/dbformat.cc
View File

@ -28,6 +28,7 @@ namespace ROCKSDB_NAMESPACE {
// ValueType, not the lowest).
const ValueType kValueTypeForSeek = kTypeDeletionWithTimestamp;
const ValueType kValueTypeForSeekForPrev = kTypeDeletion;
const std::string kDisableUserTimestamp("");
EntryType GetEntryType(ValueType value_type) {
switch (value_type) {

3
db/dbformat.h
View File

@ -93,6 +93,9 @@ static const SequenceNumber kDisableGlobalSequenceNumber = port::kMaxUint64;
constexpr uint64_t kNumInternalBytes = 8;
// Defined in dbformat.cc
extern const std::string kDisableUserTimestamp;
// The data structure that represents an internal key in the way that user_key,
// sequence number and type are stored in separated forms.
struct ParsedInternalKey {

3
db/external_sst_file_ingestion_job.cc
View File

@ -441,7 +441,8 @@ Status ExternalSstFileIngestionJob::Run() {
f.fd.GetNumber(), f.fd.GetPathId(), f.fd.GetFileSize(),
f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
f.assigned_seqno, false, kInvalidBlobFileNumber, oldest_ancester_time,
current_time, f.file_checksum, f.file_checksum_func_name);
current_time, f.file_checksum, f.file_checksum_func_name,
kDisableUserTimestamp, kDisableUserTimestamp);
f_metadata.temperature = f.file_temperature;
edit_.AddFile(f.picked_level, f_metadata);
}

3
db/flush_job.cc
View File

@ -962,7 +962,8 @@ Status FlushJob::WriteLevel0Table() {
meta_.fd.smallest_seqno, meta_.fd.largest_seqno,
meta_.marked_for_compaction, meta_.oldest_blob_file_number,
meta_.oldest_ancester_time, meta_.file_creation_time,
meta_.file_checksum, meta_.file_checksum_func_name);
meta_.file_checksum, meta_.file_checksum_func_name,
meta_.min_timestamp, meta_.max_timestamp);
edit_->SetBlobFileAdditions(std::move(blob_file_additions));
}

3
db/import_column_family_job.cc
View File

@ -155,7 +155,8 @@ Status ImportColumnFamilyJob::Run() {
f.largest_internal_key, file_metadata.smallest_seqno,
file_metadata.largest_seqno, false, kInvalidBlobFileNumber,
oldest_ancester_time, current_time, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
// If incoming sequence number is higher, update local sequence number.
if (file_metadata.largest_seqno > versions_->LastSequence()) {

3
db/repair.cc
View File

@ -635,7 +635,8 @@ class Repairer {
table->meta.fd.largest_seqno, table->meta.marked_for_compaction,
table->meta.oldest_blob_file_number,
table->meta.oldest_ancester_time, table->meta.file_creation_time,
table->meta.file_checksum, table->meta.file_checksum_func_name);
table->meta.file_checksum, table->meta.file_checksum_func_name,
table->meta.min_timestamp, table->meta.max_timestamp);
}
assert(next_file_number_ > 0);
vset_.MarkFileNumberUsed(next_file_number_ - 1);

90
db/version_builder_test.cc
View File

@ -69,7 +69,8 @@ class VersionBuilderTest : public testing::Test {
GetInternalKey(largest, largest_seq), smallest_seqno, largest_seqno,
/* marked_for_compact */ false, oldest_blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
f->compensated_file_size = file_size;
f->num_entries = num_entries;
f->num_deletions = num_deletions;
@ -130,7 +131,8 @@ class VersionBuilderTest : public testing::Test {
smallest_seqno, largest_seqno, marked_for_compaction,
blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
}
static std::shared_ptr<BlobFileMetaData> GetBlobFileMetaData(
@ -190,7 +192,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
GetInternalKey("350"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.DeleteFile(3, 27U);
EnvOptions env_options;
@ -230,7 +233,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
GetInternalKey("350"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
@ -273,7 +277,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
GetInternalKey("350"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
version_edit.DeleteFile(4, 6U);
@ -307,27 +312,32 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
GetInternalKey("350"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
GetInternalKey("450"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
GetInternalKey("650"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
GetInternalKey("550"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
GetInternalKey("750"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
EnvOptions env_options;
constexpr TableCache* table_cache = nullptr;
@ -364,27 +374,32 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
GetInternalKey("350"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
GetInternalKey("450"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
GetInternalKey("650"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
GetInternalKey("550"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
GetInternalKey("750"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
ASSERT_OK(version_builder.Apply(&version_edit));
VersionEdit version_edit2;
@ -392,14 +407,16 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
GetInternalKey("950"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
version_edit2.DeleteFile(2, 616);
version_edit2.DeleteFile(2, 636);
version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
GetInternalKey("850"), 200, 200, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
ASSERT_OK(version_builder.Apply(&version_edit2));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -500,7 +517,8 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
GetInternalKey(largest, largest_seq), smallest_seqno,
largest_seqno, marked_for_compaction, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
ASSERT_OK(builder.Apply(&addition));
@ -544,7 +562,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
smallest_seqno, largest_seqno, marked_for_compaction,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
const Status s = builder.Apply(&edit);
ASSERT_TRUE(s.IsCorruption());
@ -577,7 +596,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
ASSERT_OK(builder.Apply(&edit));
@ -590,7 +610,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
smallest_seqno, largest_seqno, marked_for_compaction,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
const Status s = builder.Apply(&other_edit);
ASSERT_TRUE(s.IsCorruption());
@ -624,7 +645,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
smallest_seqno, largest_seqno, marked_for_compaction,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
ASSERT_OK(builder.Apply(&addition));
@ -1154,7 +1176,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
GetInternalKey(smallest), GetInternalKey(largest),
smallest_seqno, largest_seqno, marked_for_compaction,
blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, checksum_value, checksum_method);
kUnknownFileCreationTime, checksum_value, checksum_method,
kDisableUserTimestamp, kDisableUserTimestamp);
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value);
@ -1238,7 +1261,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
/* largest_seqno */ 200, /* marked_for_compaction */ false,
/* oldest_blob_file_number */ 16, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
edit.AddFile(/* level */ 1, /* file_number */ 700, /* path_id */ 0,
/* file_size */ 100, /* smallest */ GetInternalKey("801"),
@ -1246,7 +1270,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
/* largest_seqno */ 200, /* marked_for_compaction */ false,
/* oldest_blob_file_number */ 1000, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
edit.AddBlobFile(/* blob_file_number */ 1000, /* total_blob_count */ 2000,
/* total_blob_bytes */ 200000,
/* checksum_method */ std::string(),
@ -1464,7 +1489,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 2100, /* marked_for_compaction */ false,
/* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
// Add an SST that does not reference any blob files.
edit.AddFile(
@ -1474,7 +1500,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 2200, /* marked_for_compaction */ false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
// Delete a file that references a blob file.
edit.DeleteFile(/* level */ 1, /* file_number */ 6);
@ -1496,7 +1523,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 300, /* marked_for_compaction */ false,
/* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
// Trivially move a file that does not reference any blob files.
edit.DeleteFile(/* level */ 1, /* file_number */ 13);
@ -1507,7 +1535,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 1300, /* marked_for_compaction */ false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
// Add one more SST file that references a blob file, then promptly
// delete it in a second version edit before the new version gets saved.
@ -1519,7 +1548,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 2300, /* marked_for_compaction */ false,
/* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
VersionEdit edit2;

29
db/version_edit.cc
View File

@ -186,6 +186,16 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));
if (f.max_timestamp != kDisableUserTimestamp) {
if (f.min_timestamp.size() != f.max_timestamp.size()) {
assert(false);
return false;
}
PutVarint32(dst, NewFileCustomTag::kMinTimestamp);
PutLengthPrefixedSlice(dst, Slice(f.min_timestamp));
PutVarint32(dst, NewFileCustomTag::kMaxTimestamp);
PutLengthPrefixedSlice(dst, Slice(f.max_timestamp));
}
if (f.fd.GetPathId() != 0) {
PutVarint32(dst, NewFileCustomTag::kPathId);
char p = static_cast<char>(f.fd.GetPathId());
@ -315,6 +325,10 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
return "new-file4 custom field";
}
if (custom_tag == kTerminate) {
if (f.min_timestamp.size() != f.max_timestamp.size()) {
assert(false);
return "new-file4 custom field timestamp size mismatch error";
}
break;
}
if (!GetLengthPrefixedSlice(input, &field)) {
@ -375,6 +389,12 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
}
}
break;
case kMinTimestamp:
f.min_timestamp = field.ToString();
break;
case kMaxTimestamp:
f.max_timestamp = field.ToString();
break;
default:
if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) {
// Should not proceed if cannot understand it
@ -781,6 +801,10 @@ std::string VersionEdit::DebugString(bool hex_key) const {
r.append(" blob_file:");
AppendNumberTo(&r, f.oldest_blob_file_number);
}
r.append(" min_timestamp:");
r.append(Slice(f.min_timestamp).ToString(true));
r.append(" max_timestamp:");
r.append(Slice(f.max_timestamp).ToString(true));
r.append(" oldest_ancester_time:");
AppendNumberTo(&r, f.oldest_ancester_time);
r.append(" file_creation_time:");
@ -894,6 +918,11 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
jw << "FileSize" << f.fd.GetFileSize();
jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
jw << "LargestIKey" << f.largest.DebugString(hex_key);
if (f.min_timestamp != kDisableUserTimestamp) {
assert(f.max_timestamp != kDisableUserTimestamp);
jw << "MinTimestamp" << Slice(f.min_timestamp).ToString(true);
jw << "MaxTimestamp" << Slice(f.max_timestamp).ToString(true);
}
if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
jw << "OldestBlobFile" << f.oldest_blob_file_number;
}

28
db/version_edit.h
View File

@ -82,6 +82,8 @@ enum NewFileCustomTag : uint32_t {
kFileChecksum = 7,
kFileChecksumFuncName = 8,
kTemperature = 9,
kMinTimestamp = 10,
kMaxTimestamp = 11,
// If this bit for the custom tag is set, opening DB should fail if
// we don't know this field.
@ -209,6 +211,10 @@ struct FileMetaData {
// File checksum function name
std::string file_checksum_func_name = kUnknownFileChecksumFuncName;
// Min (oldest) timestamp of keys in this file
std::string min_timestamp;
// Max (newest) timestamp of keys in this file
std::string max_timestamp;
FileMetaData() = default;
@ -218,7 +224,8 @@ struct FileMetaData {
const SequenceNumber& largest_seq, bool marked_for_compact,
uint64_t oldest_blob_file, uint64_t _oldest_ancester_time,
uint64_t _file_creation_time, const std::string& _file_checksum,
const std::string& _file_checksum_func_name)
const std::string& _file_checksum_func_name,
std::string _min_timestamp, std::string _max_timestamp)
: fd(file, file_path_id, file_size, smallest_seq, largest_seq),
smallest(smallest_key),
largest(largest_key),
@ -227,7 +234,9 @@ struct FileMetaData {
oldest_ancester_time(_oldest_ancester_time),
file_creation_time(_file_creation_time),
file_checksum(_file_checksum),
file_checksum_func_name(_file_checksum_func_name) {
file_checksum_func_name(_file_checksum_func_name),
min_timestamp(std::move(_min_timestamp)),
max_timestamp(std::move(_max_timestamp)) {
TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", this);
}
@ -394,14 +403,17 @@ class VersionEdit {
const SequenceNumber& largest_seqno, bool marked_for_compaction,
uint64_t oldest_blob_file_number, uint64_t oldest_ancester_time,
uint64_t file_creation_time, const std::string& file_checksum,
const std::string& file_checksum_func_name) {
const std::string& file_checksum_func_name,
const std::string& min_timestamp,
const std::string& max_timestamp) {
assert(smallest_seqno <= largest_seqno);
new_files_.emplace_back(
level, FileMetaData(file, file_path_id, file_size, smallest, largest,
smallest_seqno, largest_seqno,
marked_for_compaction, oldest_blob_file_number,
oldest_ancester_time, file_creation_time,
file_checksum, file_checksum_func_name));
level,
FileMetaData(file, file_path_id, file_size, smallest, largest,
smallest_seqno, largest_seqno, marked_for_compaction,
oldest_blob_file_number, oldest_ancester_time,
file_creation_time, file_checksum, file_checksum_func_name,
min_timestamp, max_timestamp));
}
void AddFile(int level, const FileMetaData& f) {

36
db/version_edit_test.cc
View File

@ -40,7 +40,7 @@ TEST_F(VersionEditTest, EncodeDecode) {
InternalKey("foo", kBig + 500 + i, kTypeValue),
InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
kBig + 500 + i, kBig + 600 + i, false, kInvalidBlobFileNumber,
888, 678, "234", "crc32c");
888, 678, "234", "crc32c", "123", "345");
edit.DeleteFile(4, kBig + 700 + i);
}
@ -59,21 +59,24 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123",
"234");
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "345",
"543");
edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
kBig + 602, true, kInvalidBlobFileNumber, 666, 888,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "456",
"567");
edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
kBig + 603, true, 1001, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, "678", "789");
;
edit.DeleteFile(4, 700);
@ -105,6 +108,14 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
ASSERT_EQ(kInvalidBlobFileNumber,
new_files[2].second.oldest_blob_file_number);
ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
ASSERT_EQ("123", new_files[0].second.min_timestamp);
ASSERT_EQ("234", new_files[0].second.max_timestamp);
ASSERT_EQ("345", new_files[1].second.min_timestamp);
ASSERT_EQ("543", new_files[1].second.max_timestamp);
ASSERT_EQ("456", new_files[2].second.min_timestamp);
ASSERT_EQ("567", new_files[2].second.max_timestamp);
ASSERT_EQ("678", new_files[3].second.min_timestamp);
ASSERT_EQ("789", new_files[3].second.max_timestamp);
}
TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
@ -114,11 +125,12 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123",
"234");
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, kInvalidBlobFileNumber, 686, 868, "234",
"crc32c");
"crc32c", kDisableUserTimestamp, kDisableUserTimestamp);
edit.DeleteFile(4, 700);
edit.SetComparatorName("foo");
@ -157,6 +169,10 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
ASSERT_EQ(1u, parsed.GetDeletedFiles().size());
ASSERT_EQ("123", new_files[0].second.min_timestamp);
ASSERT_EQ("234", new_files[0].second.max_timestamp);
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.min_timestamp);
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.max_timestamp);
}
TEST_F(VersionEditTest, NewFile4NotSupportedField) {
@ -166,7 +182,8 @@ TEST_F(VersionEditTest, NewFile4NotSupportedField) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
@ -196,7 +213,8 @@ TEST_F(VersionEditTest, EncodeEmptyFile) {
edit.AddFile(0, 0, 0, 0, InternalKey(), InternalKey(), 0, 0, false,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
std::string buffer;
ASSERT_TRUE(!edit.EncodeTo(&buffer));
}

3
db/version_set.cc
View File

@ -5376,7 +5376,8 @@ Status VersionSet::WriteCurrentStateToManifest(
f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->oldest_blob_file_number,
f->oldest_ancester_time, f->file_creation_time,
f->file_checksum, f->file_checksum_func_name);
f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp);
}
}

15
db/version_set_test.cc
View File

@ -46,7 +46,8 @@ class GenerateLevelFilesBriefTest : public testing::Test {
InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
largest_seq, /* marked_for_compact */ false, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
files_.push_back(f);
}
@ -154,7 +155,8 @@ class VersionStorageInfoTestBase : public testing::Test {
/* largest_seq */ 0, /* marked_for_compact */ false,
oldest_blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp);
f->compensated_file_size = file_size;
vstorage_.AddFile(level, f);
}
@ -2996,7 +2998,8 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
ASSERT_NE(0, file_size);
file_metas->emplace_back(file_num, /*file_path_id=*/0, file_size, ikey,
ikey, 0, 0, false, 0, 0, 0, kUnknownFileChecksum,
kUnknownFileChecksumFuncName);
kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
}
}
@ -3051,7 +3054,8 @@ TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
FileMetaData meta =
FileMetaData(file_num, /*file_path_id=*/0, /*file_size=*/12,
smallest_ikey, largest_ikey, 0, 0, false, 0, 0, 0,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
added_files.emplace_back(0, meta);
}
WriteFileAdditionAndDeletionToManifest(
@ -3106,7 +3110,8 @@ TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
FileMetaData meta =
FileMetaData(file_num, /*file_path_id=*/0, /*file_size=*/12,
smallest_ikey, largest_ikey, 0, 0, false, 0, 0, 0,
kUnknownFileChecksum, kUnknownFileChecksumFuncName);
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp);
added_files.emplace_back(0, meta);
}
WriteFileAdditionAndDeletionToManifest(