From 6bb7e3ef253b466baa4bd9df9ff47205f818a284 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Mon, 8 Sep 2014 22:24:40 -0700 Subject: [PATCH] Merger test Summary: I abandoned https://reviews.facebook.net/D18789, but I wrote a good unit test there, so let's check it in. :) Test Plan: this is test Reviewers: sdong, yhchiang, ljin Reviewed By: ljin Subscribers: leveldb Differential Revision: https://reviews.facebook.net/D22827 --- Makefile | 4 + table/merger_test.cc | 197 +++++++++++++++++++++++++++++++++++++++++++ util/testutil.cc | 9 ++ util/testutil.h | 2 + 4 files changed, 212 insertions(+) create mode 100644 table/merger_test.cc diff --git a/Makefile b/Makefile index da85ae2fc..a438230cb 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,7 @@ TESTS = \ manual_compaction_test \ memenv_test \ merge_test \ + merger_test \ redis_test \ reduce_levels_test \ plain_table_db_test \ @@ -434,6 +435,9 @@ write_controller_test: db/write_controller_test.o $(LIBOBJECTS) $(TESTHARNESS) merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) +merger_test: table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS) + $(CXX) table/merger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) + deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(CXX) db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) diff --git a/table/merger_test.cc b/table/merger_test.cc new file mode 100644 index 000000000..3a10527f4 --- /dev/null +++ b/table/merger_test.cc @@ -0,0 +1,197 @@ +// Copyright (c) 2013, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include +#include +#include + +#include "rocksdb/iterator.h" +#include "table/merger.h" +#include "util/testharness.h" +#include "util/testutil.h" + +namespace rocksdb { + +class VectorIterator : public Iterator { + public: + explicit VectorIterator(const std::vector& keys) + : keys_(keys), current_(keys.size()) { + std::sort(keys_.begin(), keys_.end()); + } + + virtual bool Valid() const { return current_ < keys_.size(); } + + virtual void SeekToFirst() { current_ = 0; } + virtual void SeekToLast() { current_ = keys_.size() - 1; } + + virtual void Seek(const Slice& target) { + current_ = std::lower_bound(keys_.begin(), keys_.end(), target.ToString()) - + keys_.begin(); + } + + virtual void Next() { current_++; } + virtual void Prev() { current_--; } + + virtual Slice key() const { return Slice(keys_[current_]); } + virtual Slice value() const { return Slice(); } + + virtual Status status() const { return Status::OK(); } + + private: + std::vector keys_; + size_t current_; +}; + +class MergerTest { + public: + MergerTest() + : rnd_(3), merging_iterator_(nullptr), single_iterator_(nullptr) {} + ~MergerTest() = default; + std::vector GenerateStrings(int len, int string_len) { + std::vector ret; + for (int i = 0; i < len; ++i) { + ret.push_back(test::RandomHumanReadableString(&rnd_, string_len)); + } + return ret; + } + + void AssertEquivalence() { + auto a = merging_iterator_.get(); + auto b = single_iterator_.get(); + if (!a->Valid()) { + ASSERT_TRUE(!b->Valid()); + } else { + ASSERT_TRUE(b->Valid()); + ASSERT_EQ(b->key().ToString(), a->key().ToString()); + ASSERT_EQ(b->value().ToString(), a->value().ToString()); + } + } + + void SeekToRandom() { Seek(test::RandomHumanReadableString(&rnd_, 5)); } + + void Seek(std::string target) { + merging_iterator_->Seek(target); + single_iterator_->Seek(target); + } + + void SeekToFirst() { + merging_iterator_->SeekToFirst(); + single_iterator_->SeekToFirst(); + } + + void SeekToLast() { + merging_iterator_->SeekToLast(); + single_iterator_->SeekToLast(); + } + + void Next(int times) { + for (int i = 0; i < times && merging_iterator_->Valid(); ++i) { + AssertEquivalence(); + merging_iterator_->Next(); + single_iterator_->Next(); + } + AssertEquivalence(); + } + + void Prev(int times) { + for (int i = 0; i < times && merging_iterator_->Valid(); ++i) { + AssertEquivalence(); + merging_iterator_->Prev(); + single_iterator_->Prev(); + } + AssertEquivalence(); + } + + void NextAndPrev(int times) { + for (int i = 0; i < times && merging_iterator_->Valid(); ++i) { + AssertEquivalence(); + if (rnd_.OneIn(2)) { + merging_iterator_->Prev(); + single_iterator_->Prev(); + } else { + merging_iterator_->Next(); + single_iterator_->Next(); + } + } + AssertEquivalence(); + } + + void Generate(size_t num_iterators, size_t strings_per_iterator, + size_t letters_per_string) { + std::vector small_iterators; + for (size_t i = 0; i < num_iterators; ++i) { + auto strings = GenerateStrings(strings_per_iterator, letters_per_string); + small_iterators.push_back(new VectorIterator(strings)); + all_keys_.insert(all_keys_.end(), strings.begin(), strings.end()); + } + + merging_iterator_.reset(NewMergingIterator( + BytewiseComparator(), &small_iterators[0], small_iterators.size())); + single_iterator_.reset(new VectorIterator(all_keys_)); + } + + Random rnd_; + std::unique_ptr merging_iterator_; + std::unique_ptr single_iterator_; + std::vector all_keys_; +}; + +TEST(MergerTest, SeekToRandomNextTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToRandom(); + AssertEquivalence(); + Next(50000); + } +} + +TEST(MergerTest, SeekToRandomNextSmallStringsTest) { + Generate(1000, 50, 2); + for (int i = 0; i < 10; ++i) { + SeekToRandom(); + AssertEquivalence(); + Next(50000); + } +} + +TEST(MergerTest, SeekToRandomPrevTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToRandom(); + AssertEquivalence(); + Prev(50000); + } +} + +TEST(MergerTest, SeekToRandomRandomTest) { + Generate(200, 50, 50); + for (int i = 0; i < 3; ++i) { + SeekToRandom(); + AssertEquivalence(); + NextAndPrev(5000); + } +} + +TEST(MergerTest, SeekToFirstTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToFirst(); + AssertEquivalence(); + Next(50000); + } +} + +TEST(MergerTest, SeekToLastTest) { + Generate(1000, 50, 50); + for (int i = 0; i < 10; ++i) { + SeekToLast(); + AssertEquivalence(); + Prev(50000); + } +} + +} // namespace rocksdb + +int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); } diff --git a/util/testutil.cc b/util/testutil.cc index 363b8ff19..20f22c2dc 100644 --- a/util/testutil.cc +++ b/util/testutil.cc @@ -23,6 +23,15 @@ Slice RandomString(Random* rnd, int len, std::string* dst) { return Slice(*dst); } +extern std::string RandomHumanReadableString(Random* rnd, int len) { + std::string ret; + ret.resize(len); + for (int i = 0; i < len; ++i) { + ret[i] = static_cast('a' + rnd->Uniform(26)); + } + return ret; +} + std::string RandomKey(Random* rnd, int len) { // Make sure to generate a wide variety of characters so we // test the boundary conditions for short-key optimizations. diff --git a/util/testutil.h b/util/testutil.h index c615fc1e7..eff0d7e7d 100644 --- a/util/testutil.h +++ b/util/testutil.h @@ -21,6 +21,8 @@ namespace test { // references the generated data. extern Slice RandomString(Random* rnd, int len, std::string* dst); +extern std::string RandomHumanReadableString(Random* rnd, int len); + // Return a random key with the specified length that may contain interesting // characters (e.g. \x00, \xff, etc.). extern std::string RandomKey(Random* rnd, int len);