Fix issue where IngestExternalFile insert blocks in block cache with g_seqno=0
Summary: When we Ingest an external file we open it to read some metadata and first/last key during doing that we insert blocks into the block cache with global_seqno = 0 If we move the file (did not copy it) into the DB, we will use these blocks with the wrong seqno in the read path Closes https://github.com/facebook/rocksdb/pull/1627 Differential Revision: D4293332 Pulled By: yiwu-arbug fbshipit-source-id: 3ce5523
This commit is contained in:
parent
80474e6791
commit
b1124fa127
@ -311,8 +311,14 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
|
|||||||
file_to_ingest->num_entries = props->num_entries;
|
file_to_ingest->num_entries = props->num_entries;
|
||||||
|
|
||||||
ParsedInternalKey key;
|
ParsedInternalKey key;
|
||||||
std::unique_ptr<InternalIterator> iter(
|
ReadOptions ro;
|
||||||
table_reader->NewIterator(ReadOptions()));
|
// During reading the external file we can cache blocks that we read into
|
||||||
|
// the block cache, if we later change the global seqno of this file, we will
|
||||||
|
// have block in cache that will include keys with wrong seqno.
|
||||||
|
// We need to disable fill_cache so that we read from the file without
|
||||||
|
// updating the block cache.
|
||||||
|
ro.fill_cache = false;
|
||||||
|
std::unique_ptr<InternalIterator> iter(table_reader->NewIterator(ro));
|
||||||
|
|
||||||
// Get first (smallest) key from file
|
// Get first (smallest) key from file
|
||||||
iter->SeekToFirst();
|
iter->SeekToFirst();
|
||||||
|
@ -15,7 +15,7 @@ namespace rocksdb {
|
|||||||
class ExternalSSTFileTest : public DBTestBase {
|
class ExternalSSTFileTest : public DBTestBase {
|
||||||
public:
|
public:
|
||||||
ExternalSSTFileTest() : DBTestBase("/external_sst_file_test") {
|
ExternalSSTFileTest() : DBTestBase("/external_sst_file_test") {
|
||||||
sst_files_dir_ = test::TmpDir(env_) + "/sst_files/";
|
sst_files_dir_ = dbname_ + "/sst_files/";
|
||||||
DestroyAndRecreateExternalSSTFilesDir();
|
DestroyAndRecreateExternalSSTFilesDir();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1895,6 +1895,39 @@ TEST_F(ExternalSSTFileTest, IngestionListener) {
|
|||||||
ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_name,
|
ASSERT_EQ(listener->ingested_files.back().table_properties.column_family_name,
|
||||||
"toto");
|
"toto");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(ExternalSSTFileTest, SnapshotInconsistencyBug) {
|
||||||
|
Options options = CurrentOptions();
|
||||||
|
DestroyAndReopen(options);
|
||||||
|
const int kNumKeys = 10000;
|
||||||
|
|
||||||
|
// Insert keys using normal path and take a snapshot
|
||||||
|
for (int i = 0; i < kNumKeys; i++) {
|
||||||
|
ASSERT_OK(Put(Key(i), Key(i) + "_V1"));
|
||||||
|
}
|
||||||
|
const Snapshot* snap = db_->GetSnapshot();
|
||||||
|
|
||||||
|
// Overwrite all keys using IngestExternalFile
|
||||||
|
std::string sst_file_path = sst_files_dir_ + "file1.sst";
|
||||||
|
SstFileWriter sst_file_writer(EnvOptions(), options, options.comparator);
|
||||||
|
ASSERT_OK(sst_file_writer.Open(sst_file_path));
|
||||||
|
for (int i = 0; i < kNumKeys; i++) {
|
||||||
|
ASSERT_OK(sst_file_writer.Add(Key(i), Key(i) + "_V2"));
|
||||||
|
}
|
||||||
|
ASSERT_OK(sst_file_writer.Finish());
|
||||||
|
|
||||||
|
IngestExternalFileOptions ifo;
|
||||||
|
ifo.move_files = true;
|
||||||
|
ASSERT_OK(db_->IngestExternalFile({sst_file_path}, ifo));
|
||||||
|
|
||||||
|
for (int i = 0; i < kNumKeys; i++) {
|
||||||
|
ASSERT_EQ(Get(Key(i), snap), Key(i) + "_V1");
|
||||||
|
ASSERT_EQ(Get(Key(i)), Key(i) + "_V2");
|
||||||
|
}
|
||||||
|
|
||||||
|
db_->ReleaseSnapshot(snap);
|
||||||
|
}
|
||||||
|
|
||||||
#endif // ROCKSDB_LITE
|
#endif // ROCKSDB_LITE
|
||||||
|
|
||||||
} // namespace rocksdb
|
} // namespace rocksdb
|
||||||
|
Loading…
x
Reference in New Issue
Block a user