From e79ad9e18434b3422b78f60129ec6e77bdefb5fb Mon Sep 17 00:00:00 2001 From: sdong Date: Mon, 29 Feb 2016 18:38:03 -0800 Subject: [PATCH] Add Iterator Property rocksdb.iterator.version_number Summary: We want to provide a way to detect whether an iterator is stale and needs to be recreated. Add a iterator property to return version number. Test Plan: Add two unit tests for it. Reviewers: IslamAbdelRahman, yhchiang, anthony, kradhakrishnan, andrewkr Reviewed By: andrewkr Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D54921 --- CMakeLists.txt | 1 + Makefile | 4 + db/db_impl.cc | 16 ++-- db/db_impl_readonly.cc | 16 ++-- db/db_iter.cc | 26 ++++-- db/db_iter.h | 16 ++-- db/db_iter_test.cc | 186 ++++++++++++++++++------------------- db/db_tailing_iter_test.cc | 45 +++++++++ db/db_test2.cc | 63 +++++++++++++ db/forward_iterator.cc | 16 +++- db/forward_iterator.h | 2 + include/rocksdb/iterator.h | 2 + table/internal_iterator.h | 5 + 13 files changed, 267 insertions(+), 131 deletions(-) create mode 100644 db/db_test2.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index baed60308..eda0d703b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -337,6 +337,7 @@ set(TESTS db/db_table_properties_test.cc db/db_tailing_iter_test.cc db/db_test.cc + db/db_test2.cc db/db_universal_compaction_test.cc db/db_wal_test.cc db/dbformat_test.cc diff --git a/Makefile b/Makefile index 018184c75..cfe70bea9 100644 --- a/Makefile +++ b/Makefile @@ -245,6 +245,7 @@ BENCHTOOLOBJECTS = $(BENCH_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL) TESTS = \ db_test \ + db_test2 \ db_iter_test \ db_log_iter_test \ db_compaction_filter_test \ @@ -790,6 +791,9 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS) db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) + $(AM_LINK) + db_log_iter_test: db/db_log_iter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) diff --git a/db/db_impl.cc b/db/db_impl.cc index fc8c531c4..eb46811ae 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -3922,8 +3922,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, env_, *cfd->ioptions(), cfd->user_comparator(), iter, kMaxSequenceNumber, sv->mutable_cf_options.max_sequential_skip_in_iterations, - read_options.iterate_upper_bound, read_options.prefix_same_as_start, - read_options.pin_data); + sv->version_number, read_options.iterate_upper_bound, + read_options.prefix_same_as_start, read_options.pin_data); #endif } else { SequenceNumber latest_snapshot = versions_->LastSequence(); @@ -3980,8 +3980,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator( env_, *cfd->ioptions(), cfd->user_comparator(), snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations, - read_options.iterate_upper_bound, read_options.prefix_same_as_start, - read_options.pin_data); + sv->version_number, read_options.iterate_upper_bound, + read_options.prefix_same_as_start, read_options.pin_data); InternalIterator* internal_iter = NewInternalIterator(read_options, cfd, sv, db_iter->GetArena()); @@ -4034,8 +4034,8 @@ Status DBImpl::NewIterators( iterators->push_back(NewDBIterator( env_, *cfd->ioptions(), cfd->user_comparator(), iter, kMaxSequenceNumber, - sv->mutable_cf_options.max_sequential_skip_in_iterations, nullptr, - false, read_options.pin_data)); + sv->mutable_cf_options.max_sequential_skip_in_iterations, + sv->version_number, nullptr, false, read_options.pin_data)); } #endif } else { @@ -4054,8 +4054,8 @@ Status DBImpl::NewIterators( ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator( env_, *cfd->ioptions(), cfd->user_comparator(), snapshot, - sv->mutable_cf_options.max_sequential_skip_in_iterations, nullptr, - false, read_options.pin_data); + sv->mutable_cf_options.max_sequential_skip_in_iterations, + sv->version_number, nullptr, false, read_options.pin_data); InternalIterator* internal_iter = NewInternalIterator(read_options, cfd, sv, db_iter->GetArena()); db_iter->SetIterUnderDBIter(internal_iter); diff --git a/db/db_impl_readonly.cc b/db/db_impl_readonly.cc index d96362f79..57c14df14 100644 --- a/db/db_impl_readonly.cc +++ b/db/db_impl_readonly.cc @@ -54,10 +54,11 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options, auto db_iter = NewArenaWrappedDbIterator( env_, *cfd->ioptions(), cfd->user_comparator(), (read_options.snapshot != nullptr - ? reinterpret_cast( - read_options.snapshot)->number_ + ? reinterpret_cast(read_options.snapshot) + ->number_ : latest_snapshot), - super_version->mutable_cf_options.max_sequential_skip_in_iterations); + super_version->mutable_cf_options.max_sequential_skip_in_iterations, + super_version->version_number); auto internal_iter = NewInternalIterator( read_options, cfd, super_version, db_iter->GetArena()); db_iter->SetIterUnderDBIter(internal_iter); @@ -81,10 +82,11 @@ Status DBImplReadOnly::NewIterators( auto* db_iter = NewArenaWrappedDbIterator( env_, *cfd->ioptions(), cfd->user_comparator(), (read_options.snapshot != nullptr - ? reinterpret_cast( - read_options.snapshot)->number_ - : latest_snapshot), - sv->mutable_cf_options.max_sequential_skip_in_iterations); + ? reinterpret_cast(read_options.snapshot) + ->number_ + : latest_snapshot), + sv->mutable_cf_options.max_sequential_skip_in_iterations, + sv->version_number); auto* internal_iter = NewInternalIterator( read_options, cfd, sv, db_iter->GetArena()); db_iter->SetIterUnderDBIter(internal_iter); diff --git a/db/db_iter.cc b/db/db_iter.cc index 4f4317f9a..bfde03d17 100644 --- a/db/db_iter.cc +++ b/db/db_iter.cc @@ -13,18 +13,19 @@ #include #include -#include "db/filename.h" #include "db/dbformat.h" +#include "db/filename.h" #include "port/port.h" #include "rocksdb/env.h" -#include "rocksdb/options.h" #include "rocksdb/iterator.h" #include "rocksdb/merge_operator.h" +#include "rocksdb/options.h" #include "table/internal_iterator.h" #include "util/arena.h" #include "util/logging.h" #include "util/mutexlock.h" #include "util/perf_context_imp.h" +#include "util/string_util.h" namespace rocksdb { @@ -61,7 +62,7 @@ class DBIter: public Iterator { DBIter(Env* env, const ImmutableCFOptions& ioptions, const Comparator* cmp, InternalIterator* iter, SequenceNumber s, bool arena_mode, - uint64_t max_sequential_skip_in_iterations, + uint64_t max_sequential_skip_in_iterations, uint64_t version_number, const Slice* iterate_upper_bound = nullptr, bool prefix_same_as_start = false) : arena_mode_(arena_mode), @@ -75,6 +76,7 @@ class DBIter: public Iterator { valid_(false), current_entry_is_merged_(false), statistics_(ioptions.statistics), + version_number_(version_number), iterate_upper_bound_(iterate_upper_bound), prefix_same_as_start_(prefix_same_as_start), iter_pinned_(false) { @@ -142,7 +144,13 @@ class DBIter: public Iterator { if (prop == nullptr) { return Status::InvalidArgument("prop is nullptr"); } - if (prop_name == "rocksdb.iterator.is-key-pinned") { + if (prop_name == "rocksdb.iterator.version-number") { + // First try to pass the value returned from inner iterator. + if (!iter_->GetProperty(prop_name, prop).ok()) { + *prop = ToString(version_number_); + } + return Status::OK(); + } else if (prop_name == "rocksdb.iterator.is-key-pinned") { if (valid_) { *prop = (iter_pinned_ && saved_key_.IsKeyPinned()) ? "1" : "0"; } else { @@ -198,6 +206,7 @@ class DBIter: public Iterator { bool current_entry_is_merged_; Statistics* statistics_; uint64_t max_skip_; + uint64_t version_number_; const Slice* iterate_upper_bound_; IterKey prefix_start_; bool prefix_same_as_start_; @@ -830,12 +839,13 @@ Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& ioptions, InternalIterator* internal_iter, const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations, + uint64_t version_number, const Slice* iterate_upper_bound, bool prefix_same_as_start, bool pin_data) { DBIter* db_iter = new DBIter(env, ioptions, user_key_comparator, internal_iter, sequence, - false, max_sequential_skip_in_iterations, iterate_upper_bound, - prefix_same_as_start); + false, max_sequential_skip_in_iterations, version_number, + iterate_upper_bound, prefix_same_as_start); if (pin_data) { db_iter->PinData(); } @@ -877,7 +887,7 @@ void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1, ArenaWrappedDBIter* NewArenaWrappedDbIterator( Env* env, const ImmutableCFOptions& ioptions, const Comparator* user_key_comparator, const SequenceNumber& sequence, - uint64_t max_sequential_skip_in_iterations, + uint64_t max_sequential_skip_in_iterations, uint64_t version_number, const Slice* iterate_upper_bound, bool prefix_same_as_start, bool pin_data) { ArenaWrappedDBIter* iter = new ArenaWrappedDBIter(); @@ -885,7 +895,7 @@ ArenaWrappedDBIter* NewArenaWrappedDbIterator( auto mem = arena->AllocateAligned(sizeof(DBIter)); DBIter* db_iter = new (mem) DBIter(env, ioptions, user_key_comparator, nullptr, sequence, - true, max_sequential_skip_in_iterations, + true, max_sequential_skip_in_iterations, version_number, iterate_upper_bound, prefix_same_as_start); iter->SetDBIter(db_iter); diff --git a/db/db_iter.h b/db/db_iter.h index 4060c6408..f239d2984 100644 --- a/db/db_iter.h +++ b/db/db_iter.h @@ -25,14 +25,12 @@ class InternalIterator; // Return a new iterator that converts internal keys (yielded by // "*internal_iter") that were live at the specified "sequence" number // into appropriate user keys. -extern Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& options, - const Comparator* user_key_comparator, - InternalIterator* internal_iter, - const SequenceNumber& sequence, - uint64_t max_sequential_skip_in_iterations, - const Slice* iterate_upper_bound = nullptr, - bool prefix_same_as_start = false, - bool pin_data = false); +extern Iterator* NewDBIterator( + Env* env, const ImmutableCFOptions& options, + const Comparator* user_key_comparator, InternalIterator* internal_iter, + const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations, + uint64_t version_number, const Slice* iterate_upper_bound = nullptr, + bool prefix_same_as_start = false, bool pin_data = false); // A wrapper iterator which wraps DB Iterator and the arena, with which the DB // iterator is supposed be allocated. This class is used as an entry point of @@ -78,7 +76,7 @@ class ArenaWrappedDBIter : public Iterator { extern ArenaWrappedDBIter* NewArenaWrappedDbIterator( Env* env, const ImmutableCFOptions& options, const Comparator* user_key_comparator, const SequenceNumber& sequence, - uint64_t max_sequential_skip_in_iterations, + uint64_t max_sequential_skip_in_iterations, uint64_t version_number, const Slice* iterate_upper_bound = nullptr, bool prefix_same_as_start = false, bool pin_data = false); diff --git a/db/db_iter_test.cc b/db/db_iter_test.cc index 023a0a09c..3b3030110 100644 --- a/db/db_iter_test.cc +++ b/db/db_iter_test.cc @@ -181,10 +181,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { internal_iter->AddPut("b", "val_b"); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 10, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -215,7 +214,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations)); + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -241,7 +240,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 10, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -273,7 +273,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 10, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -308,7 +309,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 10, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); db_iter->SeekToLast(); ASSERT_TRUE(!db_iter->Valid()); @@ -337,7 +339,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 7, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); SetPerfLevel(kEnableCount); ASSERT_TRUE(GetPerfLevel() == kEnableCount); @@ -374,7 +377,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 4, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 4, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -399,7 +403,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 10, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); db_iter->SeekToLast(); ASSERT_TRUE(!db_iter->Valid()); @@ -421,7 +426,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 10, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -456,7 +462,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound)); + 7, options.max_sequential_skip_in_iterations, 0, + ro.iterate_upper_bound)); SetPerfLevel(kEnableCount); ASSERT_TRUE(GetPerfLevel() == kEnableCount); @@ -482,10 +489,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { internal_iter->AddPut("b", "val_b"); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 10, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); @@ -524,10 +530,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { internal_iter->AddPut("b", "val_b"); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 2, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "b"); @@ -555,10 +560,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) { internal_iter->AddPut("c", "val_c"); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 10, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "c"); @@ -583,10 +587,9 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) { TestIterator* internal_iter = new TestIterator(BytewiseComparator()); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 0, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 0, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(!db_iter->Valid()); } @@ -595,10 +598,9 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) { TestIterator* internal_iter = new TestIterator(BytewiseComparator()); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 0, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 0, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(!db_iter->Valid()); } @@ -617,10 +619,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) { } internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 2, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2, + options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "c"); @@ -659,9 +660,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { options.statistics = rocksdb::CreateDBStatistics(); std::unique_ptr db_iter(NewDBIterator( - env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, i + 2, - options.max_sequential_skip_in_iterations)); + env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -695,9 +695,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { internal_iter->Finish(); std::unique_ptr db_iter(NewDBIterator( - env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, i + 2, - options.max_sequential_skip_in_iterations)); + env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -724,9 +723,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { internal_iter->Finish(); std::unique_ptr db_iter(NewDBIterator( - env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 202, - options.max_sequential_skip_in_iterations)); + env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, 202, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -756,10 +754,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { } internal_iter->AddPut("c", "200"); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, i, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, i, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(!db_iter->Valid()); @@ -773,10 +770,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { } internal_iter->AddPut("c", "200"); internal_iter->Finish(); - std::unique_ptr db_iter( - NewDBIterator(env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, 200, - options.max_sequential_skip_in_iterations)); + std::unique_ptr db_iter(NewDBIterator( + env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, + 200, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "c"); @@ -809,9 +805,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { internal_iter->Finish(); std::unique_ptr db_iter(NewDBIterator( - env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, i + 2, - options.max_sequential_skip_in_iterations)); + env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -844,9 +839,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) { internal_iter->Finish(); std::unique_ptr db_iter(NewDBIterator( - env_, ImmutableCFOptions(options), - BytewiseComparator(), internal_iter, i + 2, - options.max_sequential_skip_in_iterations)); + env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -887,7 +881,7 @@ TEST_F(DBIteratorTest, DBIterator1) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1, - options.max_sequential_skip_in_iterations)); + options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -913,7 +907,7 @@ TEST_F(DBIteratorTest, DBIterator2) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 0, - options.max_sequential_skip_in_iterations)); + options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -936,7 +930,7 @@ TEST_F(DBIteratorTest, DBIterator3) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2, - options.max_sequential_skip_in_iterations)); + options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -958,7 +952,7 @@ TEST_F(DBIteratorTest, DBIterator4) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 4, - options.max_sequential_skip_in_iterations)); + options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -987,7 +981,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 0, options.max_sequential_skip_in_iterations)); + 0, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1009,7 +1003,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 1, options.max_sequential_skip_in_iterations)); + 1, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1031,7 +1025,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 2, options.max_sequential_skip_in_iterations)); + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1053,7 +1047,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 3, options.max_sequential_skip_in_iterations)); + 3, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1075,7 +1069,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 4, options.max_sequential_skip_in_iterations)); + 4, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1097,7 +1091,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 5, options.max_sequential_skip_in_iterations)); + 5, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1119,7 +1113,7 @@ TEST_F(DBIteratorTest, DBIterator5) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 6, options.max_sequential_skip_in_iterations)); + 6, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1145,7 +1139,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 0, options.max_sequential_skip_in_iterations)); + 0, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1167,7 +1161,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 1, options.max_sequential_skip_in_iterations)); + 1, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1189,7 +1183,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 2, options.max_sequential_skip_in_iterations)); + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1211,7 +1205,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 3, options.max_sequential_skip_in_iterations)); + 3, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(!db_iter->Valid()); } @@ -1229,7 +1223,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 4, options.max_sequential_skip_in_iterations)); + 4, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1251,7 +1245,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 5, options.max_sequential_skip_in_iterations)); + 5, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1273,7 +1267,7 @@ TEST_F(DBIteratorTest, DBIterator6) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 6, options.max_sequential_skip_in_iterations)); + 6, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1311,7 +1305,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 0, options.max_sequential_skip_in_iterations)); + 0, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1345,7 +1339,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 2, options.max_sequential_skip_in_iterations)); + 2, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1385,7 +1379,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 4, options.max_sequential_skip_in_iterations)); + 4, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1425,7 +1419,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 5, options.max_sequential_skip_in_iterations)); + 5, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1470,7 +1464,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 6, options.max_sequential_skip_in_iterations)); + 6, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1516,7 +1510,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 7, options.max_sequential_skip_in_iterations)); + 7, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1556,7 +1550,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 9, options.max_sequential_skip_in_iterations)); + 9, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1602,7 +1596,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 13, options.max_sequential_skip_in_iterations)); + 13, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1649,7 +1643,7 @@ TEST_F(DBIteratorTest, DBIterator7) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 14, options.max_sequential_skip_in_iterations)); + 14, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1678,7 +1672,7 @@ TEST_F(DBIteratorTest, DBIterator8) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations)); + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "b"); @@ -1707,7 +1701,7 @@ TEST_F(DBIteratorTest, DBIterator9) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations)); + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); @@ -1754,7 +1748,7 @@ TEST_F(DBIteratorTest, DBIterator10) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, options.max_sequential_skip_in_iterations)); + 10, options.max_sequential_skip_in_iterations, 0)); db_iter->Seek("c"); ASSERT_TRUE(db_iter->Valid()); @@ -1778,9 +1772,9 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) { internal_iter->AddPut("b", "2"); internal_iter->Finish(); - std::unique_ptr db_iter(NewDBIterator( - env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, - 10, 0 /* force seek */)); + std::unique_ptr db_iter( + NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(), + internal_iter, 10, 0 /* force seek */, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1807,7 +1801,7 @@ TEST_F(DBIteratorTest, DBIterator11) { std::unique_ptr db_iter(NewDBIterator( env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1, - options.max_sequential_skip_in_iterations)); + options.max_sequential_skip_in_iterations, 0)); db_iter->SeekToFirst(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "a"); @@ -1832,7 +1826,7 @@ TEST_F(DBIteratorTest, DBIterator12) { std::unique_ptr db_iter( NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(), - internal_iter, 10, 0)); + internal_iter, 10, 0, 0)); db_iter->SeekToLast(); ASSERT_TRUE(db_iter->Valid()); ASSERT_EQ(db_iter->key().ToString(), "c"); @@ -1874,7 +1868,7 @@ class DBIterWithMergeIterTest : public testing::Test { db_iter_.reset(NewDBIterator(env_, ImmutableCFOptions(options_), BytewiseComparator(), merge_iter, 8 /* read data earlier than seqId 8 */, - 3 /* max iterators before reseek */)); + 3 /* max iterators before reseek */, 0)); } Env* env_; diff --git a/db/db_tailing_iter_test.cc b/db/db_tailing_iter_test.cc index 195424386..16edb64aa 100644 --- a/db/db_tailing_iter_test.cc +++ b/db/db_tailing_iter_test.cc @@ -654,6 +654,51 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorSeekToSame) { ASSERT_EQ(found, iter->key().ToString()); } +TEST_F(DBTestTailingIterator, ForwardIteratorVersionProperty) { + Options options = CurrentOptions(); + options.write_buffer_size = 1000; + + ReadOptions read_options; + read_options.tailing = true; + + Put("foo", "bar"); + + uint64_t v1, v2, v3, v4; + { + std::unique_ptr iter(db_->NewIterator(read_options)); + iter->Seek("foo"); + std::string prop_value; + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.version-number", &prop_value)); + v1 = static_cast(std::atoi(prop_value.c_str())); + + Put("foo1", "bar1"); + Flush(); + + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.version-number", &prop_value)); + v2 = static_cast(std::atoi(prop_value.c_str())); + + iter->Seek("f"); + + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.version-number", &prop_value)); + v3 = static_cast(std::atoi(prop_value.c_str())); + + ASSERT_EQ(v1, v2); + ASSERT_GT(v3, v2); + } + + { + std::unique_ptr iter(db_->NewIterator(read_options)); + iter->Seek("foo"); + std::string prop_value; + ASSERT_OK( + iter->GetProperty("rocksdb.iterator.version-number", &prop_value)); + v4 = static_cast(std::atoi(prop_value.c_str())); + } + ASSERT_EQ(v3, v4); +} } // namespace rocksdb #endif // !defined(ROCKSDB_LITE) diff --git a/db/db_test2.cc b/db/db_test2.cc new file mode 100644 index 000000000..c2b1770c7 --- /dev/null +++ b/db/db_test2.cc @@ -0,0 +1,63 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include +#include "db/db_test_util.h" +#include "port/stack_trace.h" + +namespace rocksdb { + +class DBTest2 : public DBTestBase { + public: + DBTest2() : DBTestBase("/db_test2") {} +}; + +TEST_F(DBTest2, IteratorPropertyVersionNumber) { + Put("", ""); + Iterator* iter1 = db_->NewIterator(ReadOptions()); + std::string prop_value; + ASSERT_OK(iter1->GetProperty("rocksdb.iterator.version-number", &prop_value)); + uint64_t version_number1 = + static_cast(std::atoi(prop_value.c_str())); + + Put("", ""); + Flush(); + + Iterator* iter2 = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter2->GetProperty("rocksdb.iterator.version-number", &prop_value)); + uint64_t version_number2 = + static_cast(std::atoi(prop_value.c_str())); + + ASSERT_GT(version_number2, version_number1); + + Put("", ""); + + Iterator* iter3 = db_->NewIterator(ReadOptions()); + ASSERT_OK(iter3->GetProperty("rocksdb.iterator.version-number", &prop_value)); + uint64_t version_number3 = + static_cast(std::atoi(prop_value.c_str())); + + ASSERT_EQ(version_number2, version_number3); + + iter1->SeekToFirst(); + ASSERT_OK(iter1->GetProperty("rocksdb.iterator.version-number", &prop_value)); + uint64_t version_number1_new = + static_cast(std::atoi(prop_value.c_str())); + ASSERT_EQ(version_number1, version_number1_new); + + delete iter1; + delete iter2; + delete iter3; +} +} // namespace rocksdb + +int main(int argc, char** argv) { + rocksdb::port::InstallStackTraceHandler(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/db/forward_iterator.cc b/db/forward_iterator.cc index 7af8e18af..35439c75e 100644 --- a/db/forward_iterator.cc +++ b/db/forward_iterator.cc @@ -10,15 +10,16 @@ #include #include -#include "db/job_context.h" +#include "db/column_family.h" #include "db/db_impl.h" #include "db/db_iter.h" -#include "db/column_family.h" +#include "db/dbformat.h" +#include "db/job_context.h" #include "rocksdb/env.h" #include "rocksdb/slice.h" #include "rocksdb/slice_transform.h" #include "table/merger.h" -#include "db/dbformat.h" +#include "util/string_util.h" #include "util/sync_point.h" namespace rocksdb { @@ -471,6 +472,15 @@ Status ForwardIterator::status() const { return immutable_status_; } +Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) { + assert(prop != nullptr); + if (prop_name == "rocksdb.iterator.version-number") { + *prop = ToString(sv_->version_number); + return Status::OK(); + } + return Status::InvalidArgument(); +} + void ForwardIterator::RebuildIterators(bool refresh_sv) { // Clean up Cleanup(refresh_sv); diff --git a/db/forward_iterator.h b/db/forward_iterator.h index 16a726b08..b5beeceef 100644 --- a/db/forward_iterator.h +++ b/db/forward_iterator.h @@ -71,6 +71,8 @@ class ForwardIterator : public InternalIterator { virtual Slice key() const override; virtual Slice value() const override; virtual Status status() const override; + virtual Status GetProperty(std::string prop_name, std::string* prop) override; + bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters); private: diff --git a/include/rocksdb/iterator.h b/include/rocksdb/iterator.h index 2cd107f43..7838d8912 100644 --- a/include/rocksdb/iterator.h +++ b/include/rocksdb/iterator.h @@ -105,6 +105,8 @@ class Iterator : public Cleanable { // - DB tables were created with // BlockBasedTableOptions::use_delta_encoding // set to false. + // Property "rocksdb.iterator.version-number": + // Number of LSM version used by the iterator. virtual Status GetProperty(std::string prop_name, std::string* prop); private: diff --git a/table/internal_iterator.h b/table/internal_iterator.h index 0be1c4ce2..d487b3124 100644 --- a/table/internal_iterator.h +++ b/table/internal_iterator.h @@ -6,6 +6,7 @@ #pragma once +#include #include "rocksdb/iterator.h" #include "rocksdb/status.h" @@ -78,6 +79,10 @@ class InternalIterator : public Cleanable { // set to false. virtual bool IsKeyPinned() const { return false; } + virtual Status GetProperty(std::string prop_name, std::string* prop) { + return Status::NotSupported(""); + } + private: // No copying allowed InternalIterator(const InternalIterator&) = delete;