From 3c9cef1eed347c0fb31fcaa722b526c9af01f5e7 Mon Sep 17 00:00:00 2001 From: Andres Noetzli Date: Wed, 2 Sep 2015 13:58:22 -0700 Subject: [PATCH] Unified maps with Comparator for sorting, other cleanup Summary: This diff is a collection of cleanups that were initially part of D43179. Additionally it adds a unified way of defining key-value maps that use a Comparator for sorting (this was previously implemented in four different places). Test Plan: make clean check all Reviewers: rven, anthony, yhchiang, sdong, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D45993 --- db/builder.cc | 4 +- db/compaction_job.h | 26 ++-- db/compaction_job_test.cc | 123 ++++++++---------- db/comparator_db_test.cc | 19 +-- db/db_impl.cc | 27 ++-- db/db_impl.h | 24 ++-- db/db_test.cc | 2 + db/flush_job.cc | 7 +- db/flush_job_test.cc | 4 +- db/internal_stats.cc | 49 ++++--- db/log_writer.h | 6 +- db/merge_helper.cc | 10 +- db/table_properties_collector_test.cc | 2 +- db/write_batch.cc | 8 +- .../utilities/write_batch_with_index.h | 2 +- table/meta_blocks.cc | 2 +- table/meta_blocks.h | 23 +--- table/mock_table.cc | 35 ++++- table/mock_table.h | 58 ++++----- table/table_test.cc | 120 +++++++---------- util/hash_cuckoo_rep.cc | 9 +- util/stl_wrappers.h | 46 ++++--- .../write_batch_with_index.cc | 6 +- .../write_batch_with_index_internal.h | 3 +- 24 files changed, 290 insertions(+), 325 deletions(-) diff --git a/db/builder.cc b/db/builder.cc index de9f91e61..47a1fb40f 100644 --- a/db/builder.cc +++ b/db/builder.cc @@ -137,7 +137,7 @@ Status BuildTable( // In-memory key corruption is not ok; // TODO: find a clean way to treat in memory key corruption - // Ugly walkaround to avoid compiler error for release build + // Ugly workaround to avoid compiler error for release build bool ok __attribute__((unused)) = true; ok = ParseInternalKey(key, &ikey); assert(ok); @@ -159,7 +159,7 @@ Status BuildTable( // If there are no snapshots, then this kv affect visibility at tip. // Otherwise, search though all existing snapshots to find - // the earlist snapshot that is affected by this kv. + // the earliest snapshot that is affected by this kv. SequenceNumber prev_snapshot = 0; // 0 means no previous snapshot SequenceNumber key_needs_to_exist_in_snapshot = EarliestVisibleSnapshot(ikey.sequence, snapshots, &prev_snapshot); diff --git a/db/compaction_job.h b/db/compaction_job.h index 375c6e6cc..fc0dc471d 100644 --- a/db/compaction_job.h +++ b/db/compaction_job.h @@ -10,35 +10,35 @@ #include #include +#include #include #include +#include #include #include -#include -#include -#include "db/dbformat.h" -#include "db/log_writer.h" #include "db/column_family.h" -#include "db/version_edit.h" +#include "db/dbformat.h" +#include "db/flush_scheduler.h" +#include "db/internal_stats.h" +#include "db/job_context.h" +#include "db/log_writer.h" #include "db/memtable_list.h" +#include "db/version_edit.h" +#include "db/write_controller.h" +#include "db/write_thread.h" #include "port/port.h" +#include "rocksdb/compaction_filter.h" +#include "rocksdb/compaction_job_stats.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/memtablerep.h" -#include "rocksdb/compaction_filter.h" -#include "rocksdb/compaction_job_stats.h" #include "rocksdb/transaction_log.h" #include "util/autovector.h" #include "util/event_logger.h" +#include "util/scoped_arena_iterator.h" #include "util/stop_watch.h" #include "util/thread_local.h" -#include "util/scoped_arena_iterator.h" -#include "db/internal_stats.h" -#include "db/write_controller.h" -#include "db/flush_scheduler.h" -#include "db/write_thread.h" -#include "db/job_context.h" namespace rocksdb { diff --git a/db/compaction_job_test.cc b/db/compaction_job_test.cc index a3360e822..2d0568585 100644 --- a/db/compaction_job_test.cc +++ b/db/compaction_job_test.cc @@ -92,7 +92,7 @@ class CompactionJobTest : public testing::Test { return InternalKey(user_key, seq_num, t).Encode().ToString(); } - void AddMockFile(const mock::MockFileContents& contents, int level = 0) { + void AddMockFile(const stl_wrappers::KVMap& contents, int level = 0) { assert(contents.size() > 0); bool first_key = true; @@ -143,8 +143,8 @@ class CompactionJobTest : public testing::Test { } // returns expected result after compaction - mock::MockFileContents CreateTwoFiles(bool gen_corrupted_keys) { - mock::MockFileContents expected_results; + stl_wrappers::KVMap CreateTwoFiles(bool gen_corrupted_keys) { + auto expected_results = mock::MakeMockFile(); const int kKeysPerFile = 10000; const int kCorruptKeysPerFile = 200; const int kMatchingKeys = kKeysPerFile / 2; @@ -155,7 +155,7 @@ class CompactionJobTest : public testing::Test { }; for (int i = 0; i < 2; ++i) { - mock::MockFileContents contents; + auto contents = mock::MakeMockFile(); for (int k = 0; k < kKeysPerFile; ++k) { auto key = ToString(i * kMatchingKeys + k); auto value = ToString(i * kKeysPerFile + k); @@ -215,7 +215,7 @@ class CompactionJobTest : public testing::Test { } void RunCompaction(const std::vector>& input_files, - const mock::MockFileContents& expected_results) { + const stl_wrappers::KVMap& expected_results) { auto cfd = versions_->GetColumnFamilySet()->GetDefault(); size_t num_input_files = 0; @@ -303,21 +303,16 @@ TEST_F(CompactionJobTest, SimpleCorrupted) { TEST_F(CompactionJobTest, SimpleDeletion) { NewDB(); - mock::MockFileContents file1 = { - { KeyStr("c", 4U, kTypeDeletion), "" }, - { KeyStr("c", 3U, kTypeValue), "val" } - }; + auto file1 = mock::MakeMockFile({{KeyStr("c", 4U, kTypeDeletion), ""}, + {KeyStr("c", 3U, kTypeValue), "val"}}); AddMockFile(file1); - mock::MockFileContents file2 = { - { KeyStr("b", 2U, kTypeValue), "val" }, - { KeyStr("b", 1U, kTypeValue), "val" } - }; + auto file2 = mock::MakeMockFile({{KeyStr("b", 2U, kTypeValue), "val"}, + {KeyStr("b", 1U, kTypeValue), "val"}}); AddMockFile(file2); - mock::MockFileContents expected_results = { - { KeyStr("b", 0U, kTypeValue), "val" } - }; + auto expected_results = + mock::MakeMockFile({{KeyStr("b", 0U, kTypeValue), "val"}}); SetLastSequence(4U); auto files = cfd_->current()->storage_info()->LevelFiles(0); @@ -327,22 +322,19 @@ TEST_F(CompactionJobTest, SimpleDeletion) { TEST_F(CompactionJobTest, SimpleOverwrite) { NewDB(); - mock::MockFileContents file1 = { - { KeyStr("a", 3U, kTypeValue), "val2" }, - { KeyStr("b", 4U, kTypeValue), "val3" }, - }; + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 3U, kTypeValue), "val2"}, + {KeyStr("b", 4U, kTypeValue), "val3"}, + }); AddMockFile(file1); - mock::MockFileContents file2 = { - { KeyStr("a", 1U, kTypeValue), "val" }, - { KeyStr("b", 2U, kTypeValue), "val" } - }; + auto file2 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}, + {KeyStr("b", 2U, kTypeValue), "val"}}); AddMockFile(file2); - mock::MockFileContents expected_results = { - { KeyStr("a", 0U, kTypeValue), "val2" }, - { KeyStr("b", 0U, kTypeValue), "val3" } - }; + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"}, + {KeyStr("b", 0U, kTypeValue), "val3"}}); SetLastSequence(4U); auto files = cfd_->current()->storage_info()->LevelFiles(0); @@ -352,30 +344,25 @@ TEST_F(CompactionJobTest, SimpleOverwrite) { TEST_F(CompactionJobTest, SimpleNonLastLevel) { NewDB(); - mock::MockFileContents file1 = { - { KeyStr("a", 5U, kTypeValue), "val2" }, - { KeyStr("b", 6U, kTypeValue), "val3" }, - }; + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("b", 6U, kTypeValue), "val3"}, + }); AddMockFile(file1); - mock::MockFileContents file2 = { - { KeyStr("a", 3U, kTypeValue), "val" }, - { KeyStr("b", 4U, kTypeValue), "val" } - }; + auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"}, + {KeyStr("b", 4U, kTypeValue), "val"}}); AddMockFile(file2, 1); - mock::MockFileContents file3 = { - { KeyStr("a", 1U, kTypeValue), "val" }, - { KeyStr("b", 2U, kTypeValue), "val" } - }; + auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}, + {KeyStr("b", 2U, kTypeValue), "val"}}); AddMockFile(file3, 2); // Because level 1 is not the last level, the sequence numbers of a and b // cannot be set to 0 - mock::MockFileContents expected_results = { - { KeyStr("a", 5U, kTypeValue), "val2" }, - { KeyStr("b", 6U, kTypeValue), "val3" } - }; + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"}, + {KeyStr("b", 6U, kTypeValue), "val3"}}); SetLastSequence(6U); auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0); @@ -387,23 +374,20 @@ TEST_F(CompactionJobTest, SimpleMerge) { auto merge_op = MergeOperators::CreateStringAppendOperator(); NewDB(merge_op); - mock::MockFileContents file1 = { - { KeyStr("a", 5U, kTypeMerge), "5" }, - { KeyStr("a", 4U, kTypeMerge), "4" }, - { KeyStr("a", 3U, kTypeValue), "3" }, - }; + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeMerge), "5"}, + {KeyStr("a", 4U, kTypeMerge), "4"}, + {KeyStr("a", 3U, kTypeValue), "3"}, + }); AddMockFile(file1); - mock::MockFileContents file2 = { - { KeyStr("b", 2U, kTypeMerge), "2" }, - { KeyStr("b", 1U, kTypeValue), "1" } - }; + auto file2 = mock::MakeMockFile( + {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeValue), "1"}}); AddMockFile(file2); - mock::MockFileContents expected_results = { - { KeyStr("a", 0U, kTypeValue), "3,4,5" }, - { KeyStr("b", 0U, kTypeValue), "1,2" } - }; + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"}, + {KeyStr("b", 0U, kTypeValue), "1,2"}}); SetLastSequence(5U); auto files = cfd_->current()->storage_info()->LevelFiles(0); @@ -414,24 +398,21 @@ TEST_F(CompactionJobTest, NonAssocMerge) { auto merge_op = MergeOperators::CreateStringAppendTESTOperator(); NewDB(merge_op); - mock::MockFileContents file1 = { - { KeyStr("a", 5U, kTypeMerge), "5" }, - { KeyStr("a", 4U, kTypeMerge), "4" }, - { KeyStr("a", 3U, kTypeMerge), "3" }, - }; + auto file1 = mock::MakeMockFile({ + {KeyStr("a", 5U, kTypeMerge), "5"}, + {KeyStr("a", 4U, kTypeMerge), "4"}, + {KeyStr("a", 3U, kTypeMerge), "3"}, + }); AddMockFile(file1); - mock::MockFileContents file2 = { - { KeyStr("b", 2U, kTypeMerge), "2" }, - { KeyStr("b", 1U, kTypeMerge), "1" } - }; + auto file2 = mock::MakeMockFile( + {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeMerge), "1"}}); AddMockFile(file2); - mock::MockFileContents expected_results = { - { KeyStr("a", 0U, kTypeValue), "3,4,5" }, - { KeyStr("b", 2U, kTypeMerge), "2" }, - { KeyStr("b", 1U, kTypeMerge), "1" } - }; + auto expected_results = + mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"}, + {KeyStr("b", 2U, kTypeMerge), "2"}, + {KeyStr("b", 1U, kTypeMerge), "1"}}); SetLastSequence(5U); auto files = cfd_->current()->storage_info()->LevelFiles(0); diff --git a/db/comparator_db_test.cc b/db/comparator_db_test.cc index c9f88d61e..cb944a76a 100644 --- a/db/comparator_db_test.cc +++ b/db/comparator_db_test.cc @@ -10,6 +10,7 @@ #include "rocksdb/db.h" #include "rocksdb/env.h" #include "util/hash.h" +#include "util/stl_wrappers.h" #include "util/string_util.h" #include "util/testharness.h" #include "util/testutil.h" @@ -22,18 +23,10 @@ namespace { static const Comparator* comparator; -// A comparator for std::map, using comparator -struct MapComparator { - bool operator()(const std::string& a, const std::string& b) const { - return comparator->Compare(a, b) < 0; - } -}; - -typedef std::map KVMap; - class KVIter : public Iterator { public: - explicit KVIter(const KVMap* map) : map_(map), iter_(map_->end()) {} + explicit KVIter(const stl_wrappers::KVMap* map) + : map_(map), iter_(map_->end()) {} virtual bool Valid() const override { return iter_ != map_->end(); } virtual void SeekToFirst() override { iter_ = map_->begin(); } virtual void SeekToLast() override { @@ -60,8 +53,8 @@ class KVIter : public Iterator { virtual Status status() const override { return Status::OK(); } private: - const KVMap* const map_; - KVMap::const_iterator iter_; + const stl_wrappers::KVMap* const map_; + stl_wrappers::KVMap::const_iterator iter_; }; void AssertItersEqual(Iterator* iter1, Iterator* iter2) { @@ -77,7 +70,7 @@ void AssertItersEqual(Iterator* iter1, Iterator* iter2) { void DoRandomIteraratorTest(DB* db, std::vector source_strings, Random* rnd, int num_writes, int num_iter_ops, int num_trigger_flush) { - KVMap map; + stl_wrappers::KVMap map((stl_wrappers::LessOfComparator(comparator))); for (int i = 0; i < num_writes; i++) { if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) { diff --git a/db/db_impl.cc b/db/db_impl.cc index db04e29f7..7f7fc3392 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -14,25 +14,27 @@ #endif #include +#include + #include #include #include #include #include -#include #include -#include #include +#include #include #include #include "db/builder.h" -#include "db/flush_job.h" #include "db/compaction_job.h" #include "db/db_iter.h" #include "db/dbformat.h" #include "db/event_helpers.h" #include "db/filename.h" +#include "db/flush_job.h" +#include "db/forward_iterator.h" #include "db/job_context.h" #include "db/log_reader.h" #include "db/log_writer.h" @@ -43,24 +45,23 @@ #include "db/merge_helper.h" #include "db/table_cache.h" #include "db/table_properties_collector.h" -#include "db/forward_iterator.h" #include "db/transaction_log_impl.h" #include "db/version_set.h" -#include "db/writebuffer.h" #include "db/write_batch_internal.h" #include "db/write_callback.h" +#include "db/writebuffer.h" +#include "port/likely.h" #include "port/port.h" #include "rocksdb/cache.h" -#include "port/likely.h" #include "rocksdb/compaction_filter.h" #include "rocksdb/db.h" #include "rocksdb/delete_scheduler.h" #include "rocksdb/env.h" #include "rocksdb/merge_operator.h" -#include "rocksdb/version.h" #include "rocksdb/statistics.h" #include "rocksdb/status.h" #include "rocksdb/table.h" +#include "rocksdb/version.h" #include "table/block.h" #include "table/block_based_table_factory.h" #include "table/merger.h" @@ -75,16 +76,16 @@ #include "util/db_info_dumper.h" #include "util/file_reader_writer.h" #include "util/file_util.h" -#include "util/hash_skiplist_rep.h" #include "util/hash_linklist_rep.h" -#include "util/logging.h" +#include "util/hash_skiplist_rep.h" +#include "util/iostats_context_imp.h" #include "util/log_buffer.h" +#include "util/logging.h" #include "util/mutexlock.h" #include "util/perf_context_imp.h" -#include "util/iostats_context_imp.h" #include "util/stop_watch.h" -#include "util/sync_point.h" #include "util/string_util.h" +#include "util/sync_point.h" #include "util/thread_status_updater.h" #include "util/thread_status_util.h" #include "util/xfunc.h" @@ -3274,8 +3275,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options, // | | | // +-------+-----------------------+ // - // ArenaWrappedDBIter inlines an arena area where all the iterartor in the - // the iterator tree is allocated in the order of being accessed when + // ArenaWrappedDBIter inlines an arena area where all the iterators in + // the iterator tree are allocated in the order of being accessed when // querying. // Laying out the iterators in the order of being accessed makes it more // likely that any iterator pointer is close to the iterator it points to so diff --git a/db/db_impl.h b/db/db_impl.h index 6a7d1c3f5..662c613a9 100644 --- a/db/db_impl.h +++ b/db/db_impl.h @@ -11,21 +11,25 @@ #include #include #include +#include +#include #include -#include -#include -#include -#include #include +#include +#include -#include "db/dbformat.h" #include "db/column_family.h" #include "db/compaction_job.h" +#include "db/dbformat.h" #include "db/flush_job.h" +#include "db/flush_scheduler.h" +#include "db/internal_stats.h" #include "db/log_writer.h" #include "db/snapshot_impl.h" #include "db/version_edit.h" #include "db/wal_manager.h" +#include "db/write_controller.h" +#include "db/write_thread.h" #include "db/writebuffer.h" #include "memtable_list.h" #include "port/port.h" @@ -36,15 +40,11 @@ #include "util/autovector.h" #include "util/event_logger.h" #include "util/hash.h" -#include "util/stop_watch.h" -#include "util/thread_local.h" -#include "util/scoped_arena_iterator.h" #include "util/hash.h" #include "util/instrumented_mutex.h" -#include "db/internal_stats.h" -#include "db/write_controller.h" -#include "db/flush_scheduler.h" -#include "db/write_thread.h" +#include "util/scoped_arena_iterator.h" +#include "util/stop_watch.h" +#include "util/thread_local.h" namespace rocksdb { diff --git a/db/db_test.cc b/db/db_test.cc index be295b356..9fb606f5a 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -82,6 +82,7 @@ static long TestGetTickerCount(const Options& options, Tickers ticker_type) { // `GetPropertiesOfAllTablesTest` is correct. // This test assumes entries size is different for each of the tables. namespace { + void VerifyTableProperties(DB* db, uint64_t expected_entries_size) { TablePropertiesCollection props; ASSERT_OK(db->GetPropertiesOfAllTables(&props)); @@ -110,6 +111,7 @@ uint64_t GetNumberOfSstFilesForColumnFamily(DB* db, } return result; } + } // namespace class DBTest : public DBTestBase { diff --git a/db/flush_job.cc b/db/flush_job.cc index 20cf7f860..936c375f6 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -14,6 +14,7 @@ #endif #include + #include #include @@ -28,8 +29,8 @@ #include "db/memtable_list.h" #include "db/merge_context.h" #include "db/version_set.h" -#include "port/port.h" #include "port/likely.h" +#include "port/port.h" #include "rocksdb/db.h" #include "rocksdb/env.h" #include "rocksdb/statistics.h" @@ -43,11 +44,11 @@ #include "util/coding.h" #include "util/event_logger.h" #include "util/file_util.h" -#include "util/logging.h" +#include "util/iostats_context_imp.h" #include "util/log_buffer.h" +#include "util/logging.h" #include "util/mutexlock.h" #include "util/perf_context_imp.h" -#include "util/iostats_context_imp.h" #include "util/stop_watch.h" #include "util/sync_point.h" #include "util/thread_status_util.h" diff --git a/db/flush_job_test.cc b/db/flush_job_test.cc index 1c6fe8482..d2c423c36 100644 --- a/db/flush_job_test.cc +++ b/db/flush_job_test.cc @@ -104,7 +104,7 @@ TEST_F(FlushJobTest, NonEmpty) { auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber); new_mem->Ref(); - mock::MockFileContents inserted_keys; + auto inserted_keys = mock::MakeMockFile(); // Test data: // seqno [ 1, 2 ... 8998, 8999, 9000, 9001, 9002 ... 9999 ] // key [ 1001, 1002 ... 9998, 9999, 0, 1, 2 ... 999 ] @@ -165,7 +165,7 @@ TEST_F(FlushJobTest, Snapshots) { new_mem->Ref(); SequenceNumber current_seqno = 0; - mock::MockFileContents inserted_keys; + auto inserted_keys = mock::MakeMockFile(); for (int i = 1; i < keys; ++i) { std::string key(ToString(i)); int insertions = rnd.Uniform(max_inserts_per_keys); diff --git a/db/internal_stats.cc b/db/internal_stats.cc index 10aae13dc..c9d0b9bf4 100644 --- a/db/internal_stats.cc +++ b/db/internal_stats.cc @@ -27,6 +27,7 @@ namespace rocksdb { namespace { const double kMB = 1048576.0; const double kGB = kMB * 1024; +const double kMicrosInSec = 1000000.0; void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) { snprintf( @@ -50,7 +51,7 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name, stats.bytes_read_non_output_levels + stats.bytes_read_output_level; int64_t bytes_new = stats.bytes_written - stats.bytes_read_output_level; - double elapsed = (stats.micros + 1) / 1000000.0; + double elapsed = (stats.micros + 1) / kMicrosInSec; std::string num_input_records = NumberToHumanString(stats.num_input_records); std::string num_dropped_records = NumberToHumanString(stats.num_dropped_records); @@ -73,16 +74,15 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name, " " /* Stall(cnt) */ "%7s " /* KeyIn */ "%6s\n", /* KeyDrop */ - name.c_str(), num_files, being_compacted, total_file_size / kMB, - score, bytes_read / kGB, stats.bytes_read_non_output_levels / kGB, + name.c_str(), + num_files, being_compacted, total_file_size / kMB, score, + bytes_read / kGB, stats.bytes_read_non_output_levels / kGB, stats.bytes_read_output_level / kGB, stats.bytes_written / kGB, - bytes_new / kGB, stats.bytes_moved / kGB, - w_amp, bytes_read / kMB / elapsed, - stats.bytes_written / kMB / elapsed, stats.micros / 1000000.0, - stats.count, - stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count, - stalls, - num_input_records.c_str(), num_dropped_records.c_str()); + bytes_new / kGB, stats.bytes_moved / kGB, w_amp, + bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed, + stats.micros / kMicrosInSec, stats.count, + stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count, + stalls, num_input_records.c_str(), num_dropped_records.c_str()); } } @@ -465,7 +465,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type, void InternalStats::DumpDBStats(std::string* value) { char buf[1000]; // DB-level stats, only available from default column family - double seconds_up = (env_->NowMicros() - started_at_ + 1) / 1000000.0; + double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec; double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up; snprintf(buf, sizeof(buf), "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n", @@ -524,11 +524,9 @@ void InternalStats::DumpDBStats(std::string* value) { snprintf(buf, sizeof(buf), "Cumulative compaction: %.2f GB write, %.2f MB/s write, " "%.2f GB read, %.2f MB/s read, %.1f seconds\n", - compact_bytes_write / kGB, - compact_bytes_write / kMB / seconds_up, - compact_bytes_read / kGB, - compact_bytes_read / kMB / seconds_up, - compact_micros / 1000000.0); + compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up, + compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up, + compact_micros / kMicrosInSec); value->append(buf); // Stall AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true); @@ -582,16 +580,15 @@ void InternalStats::DumpDBStats(std::string* value) { uint64_t interval_compact_micros = compact_micros - db_stats_snapshot_.compact_micros; - snprintf(buf, sizeof(buf), - "Interval compaction: %.2f GB write, %.2f MB/s write, " - "%.2f GB read, %.2f MB/s read, %.1f seconds\n", - interval_compact_bytes_write / kGB, - interval_compact_bytes_write / kMB / - std::max(interval_seconds_up, 0.001), - interval_compact_bytes_read / kGB, - interval_compact_bytes_read / kMB / - std::max(interval_seconds_up, 0.001), - interval_compact_micros / 1000000.0); + snprintf( + buf, sizeof(buf), + "Interval compaction: %.2f GB write, %.2f MB/s write, " + "%.2f GB read, %.2f MB/s read, %.1f seconds\n", + interval_compact_bytes_write / kGB, + interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001), + interval_compact_bytes_read / kGB, + interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001), + interval_compact_micros / kMicrosInSec); value->append(buf); // Stall diff --git a/db/log_writer.h b/db/log_writer.h index 8f0728c69..6b59bbdd5 100644 --- a/db/log_writer.h +++ b/db/log_writer.h @@ -6,10 +6,12 @@ // Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. - #pragma once -#include + #include + +#include + #include "db/log_format.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" diff --git a/db/merge_helper.cc b/db/merge_helper.cc index 3e8c3e5cf..427013806 100644 --- a/db/merge_helper.cc +++ b/db/merge_helper.cc @@ -2,7 +2,6 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -// #include "db/merge_helper.h" @@ -92,15 +91,12 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before, assert(!"corrupted internal key is not expected"); } break; - } - - if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) != 0) { + } else if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) != + 0) { // hit a different user key, stop right here hit_the_next_user_key = true; break; - } - - if (stop_before && ikey.sequence <= stop_before) { + } else if (stop_before && ikey.sequence <= stop_before) { // hit an entry that's visible by the previous snapshot, can't touch that break; } diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index 403de6e25..f9ebdcd2b 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -11,8 +11,8 @@ #include "db/db_impl.h" #include "db/dbformat.h" #include "db/table_properties_collector.h" -#include "rocksdb/table.h" #include "rocksdb/immutable_options.h" +#include "rocksdb/table.h" #include "table/block_based_table_factory.h" #include "table/meta_blocks.h" #include "table/plain_table_factory.h" diff --git a/db/write_batch.cc b/db/write_batch.cc index 28b2b6fc6..f79082ac5 100644 --- a/db/write_batch.cc +++ b/db/write_batch.cc @@ -27,16 +27,16 @@ #include #include -#include "rocksdb/merge_operator.h" -#include "db/dbformat.h" -#include "db/db_impl.h" #include "db/column_family.h" +#include "db/db_impl.h" +#include "db/dbformat.h" #include "db/memtable.h" #include "db/snapshot_impl.h" #include "db/write_batch_internal.h" +#include "rocksdb/merge_operator.h" #include "util/coding.h" -#include "util/statistics.h" #include "util/perf_context_imp.h" +#include "util/statistics.h" namespace rocksdb { diff --git a/include/rocksdb/utilities/write_batch_with_index.h b/include/rocksdb/utilities/write_batch_with_index.h index c1d27c17e..402e3f3a7 100644 --- a/include/rocksdb/utilities/write_batch_with_index.h +++ b/include/rocksdb/utilities/write_batch_with_index.h @@ -8,9 +8,9 @@ // // A WriteBatchWithIndex with a binary searchable index built for all the keys // inserted. +#pragma once #ifndef ROCKSDB_LITE -#pragma once #include diff --git a/table/meta_blocks.cc b/table/meta_blocks.cc index ad04b670f..7bcdf7576 100644 --- a/table/meta_blocks.cc +++ b/table/meta_blocks.cc @@ -8,9 +8,9 @@ #include #include "db/table_properties_collector.h" -#include "table/block.h" #include "rocksdb/table.h" #include "rocksdb/table_properties.h" +#include "table/block.h" #include "table/format.h" #include "table/table_properties_internal.h" #include "util/coding.h" diff --git a/table/meta_blocks.h b/table/meta_blocks.h index 167ba5900..005bcaae2 100644 --- a/table/meta_blocks.h +++ b/table/meta_blocks.h @@ -6,8 +6,8 @@ #include #include -#include #include +#include #include "db/builder.h" #include "db/table_properties_collector.h" @@ -16,6 +16,7 @@ #include "rocksdb/slice.h" #include "table/block_builder.h" #include "table/format.h" +#include "util/stl_wrappers.h" namespace rocksdb { @@ -27,22 +28,6 @@ class Logger; class RandomAccessFile; struct TableProperties; -// An STL style comparator that does the bytewise comparator comparasion -// internally. -struct BytewiseLessThan { - bool operator()(const std::string& key1, const std::string& key2) const { - // smaller entries will be placed in front. - return comparator->Compare(key1, key2) <= 0; - } - - const Comparator* comparator = BytewiseComparator(); -}; - -// When writing to a block that requires entries to be sorted by -// `BytewiseComparator`, we can buffer the content to `BytewiseSortedMap` -// before writng to store. -typedef std::map BytewiseSortedMap; - class MetaIndexBuilder { public: MetaIndexBuilder(const MetaIndexBuilder&) = delete; @@ -57,7 +42,7 @@ class MetaIndexBuilder { private: // store the sorted key/handle of the metablocks. - BytewiseSortedMap meta_block_handles_; + stl_wrappers::KVMap meta_block_handles_; std::unique_ptr meta_index_block_; }; @@ -78,7 +63,7 @@ class PropertyBlockBuilder { private: std::unique_ptr properties_block_; - BytewiseSortedMap props_; + stl_wrappers::KVMap props_; }; // Were we encounter any error occurs during user-defined statistics collection, diff --git a/table/mock_table.cc b/table/mock_table.cc index c62a61920..00bedec59 100644 --- a/table/mock_table.cc +++ b/table/mock_table.cc @@ -5,17 +5,29 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. +#include "table/mock_table.h" + #include "db/dbformat.h" #include "port/port.h" #include "rocksdb/table_properties.h" #include "table/get_context.h" -#include "table/mock_table.h" #include "util/coding.h" #include "util/file_reader_writer.h" namespace rocksdb { namespace mock { +namespace { + +const InternalKeyComparator icmp_(BytewiseComparator()); + +} // namespace + +stl_wrappers::KVMap MakeMockFile( + std::initializer_list> l) { + return stl_wrappers::KVMap(l, stl_wrappers::LessOfComparator(&icmp_)); +} + Iterator* MockTableReader::NewIterator(const ReadOptions&, Arena* arena) { return new MockTableIterator(table_); } @@ -71,7 +83,7 @@ TableBuilder* MockTableFactory::NewTableBuilder( } Status MockTableFactory::CreateMockTable(Env* env, const std::string& fname, - MockFileContents file_contents) { + stl_wrappers::KVMap file_contents) { std::unique_ptr file; auto s = env->NewWritableFile(fname, &file, EnvOptions()); if (!s.ok()) { @@ -99,16 +111,29 @@ uint32_t MockTableFactory::GetIDFromFile(RandomAccessFileReader* file) const { return DecodeFixed32(buf); } -void MockTableFactory::AssertSingleFile(const MockFileContents& file_contents) { +void MockTableFactory::AssertSingleFile( + const stl_wrappers::KVMap& file_contents) { ASSERT_EQ(file_system_.files.size(), 1U); ASSERT_TRUE(file_contents == file_system_.files.begin()->second); } -void MockTableFactory::AssertLatestFile(const MockFileContents& file_contents) { +void MockTableFactory::AssertLatestFile( + const stl_wrappers::KVMap& file_contents) { ASSERT_GE(file_system_.files.size(), 1U); auto latest = file_system_.files.end(); --latest; - ASSERT_TRUE(file_contents == latest->second); + + if (file_contents != latest->second) { + std::cout << "Wrong content! Content of latest file:" << std::endl; + for (const auto& kv : latest->second) { + ParsedInternalKey ikey; + std::string key, value; + std::tie(key, value) = kv; + ParseInternalKey(Slice(key), &ikey); + std::cout << ikey.DebugString(false) << " -> " << value << std::endl; + } + ASSERT_TRUE(false); + } } } // namespace mock diff --git a/table/mock_table.h b/table/mock_table.h index 214bcef96..d57e26e8b 100644 --- a/table/mock_table.h +++ b/table/mock_table.h @@ -5,51 +5,39 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. #pragma once + #include -#include -#include #include #include +#include +#include #include +#include +#include "port/port.h" #include "rocksdb/comparator.h" #include "rocksdb/table.h" -#include "table/table_reader.h" #include "table/table_builder.h" -#include "port/port.h" +#include "table/table_reader.h" #include "util/mutexlock.h" +#include "util/stl_wrappers.h" #include "util/testharness.h" #include "util/testutil.h" namespace rocksdb { namespace mock { -struct MockFileContentsCmp { - MockFileContentsCmp() : icmp_(BytewiseComparator()) {} - - bool operator() (const std::string& x, const std::string& y) const { - InternalKey ikey_x; - InternalKey ikey_y; - ikey_x.DecodeFrom(x); - ikey_y.DecodeFrom(y); - return icmp_.Compare(ikey_x, ikey_y) < 0; - } - - InternalKeyComparator icmp_; -}; - -// NOTE: this currently only supports the bytewise comparator -typedef std::map - MockFileContents; +stl_wrappers::KVMap MakeMockFile( + std::initializer_list> l = {}); struct MockTableFileSystem { port::Mutex mutex; - std::map files; + std::map files; }; class MockTableReader : public TableReader { public: - explicit MockTableReader(const MockFileContents& table) : table_(table) {} + explicit MockTableReader(const stl_wrappers::KVMap& table) : table_(table) {} Iterator* NewIterator(const ReadOptions&, Arena* arena) override; @@ -67,12 +55,12 @@ class MockTableReader : public TableReader { ~MockTableReader() {} private: - const MockFileContents& table_; + const stl_wrappers::KVMap& table_; }; class MockTableIterator : public Iterator { public: - explicit MockTableIterator(const MockFileContents& table) : table_(table) { + explicit MockTableIterator(const stl_wrappers::KVMap& table) : table_(table) { itr_ = table_.end(); } @@ -107,14 +95,16 @@ class MockTableIterator : public Iterator { Status status() const override { return Status::OK(); } private: - const MockFileContents& table_; - MockFileContents::const_iterator itr_; + const stl_wrappers::KVMap& table_; + stl_wrappers::KVMap::const_iterator itr_; }; class MockTableBuilder : public TableBuilder { public: MockTableBuilder(uint32_t id, MockTableFileSystem* file_system) - : id_(id), file_system_(file_system) {} + : id_(id), file_system_(file_system) { + table_ = MakeMockFile({}); + } // REQUIRES: Either Finish() or Abandon() has been called. ~MockTableBuilder() {} @@ -148,7 +138,7 @@ class MockTableBuilder : public TableBuilder { private: uint32_t id_; MockTableFileSystem* file_system_; - MockFileContents table_; + stl_wrappers::KVMap table_; }; class MockTableFactory : public TableFactory { @@ -166,10 +156,10 @@ class MockTableFactory : public TableFactory { WritableFileWriter* file) const override; // This function will directly create mock table instead of going through - // MockTableBuilder. MockFileContents has to have a format of . Those key-value pairs will then be inserted into the mock table + // MockTableBuilder. file_contents has to have a format of . Those key-value pairs will then be inserted into the mock table. Status CreateMockTable(Env* env, const std::string& fname, - MockFileContents file_contents); + stl_wrappers::KVMap file_contents); virtual Status SanitizeOptions( const DBOptions& db_opts, @@ -183,8 +173,8 @@ class MockTableFactory : public TableFactory { // This function will assert that only a single file exists and that the // contents are equal to file_contents - void AssertSingleFile(const MockFileContents& file_contents); - void AssertLatestFile(const MockFileContents& file_contents); + void AssertSingleFile(const stl_wrappers::KVMap& file_contents); + void AssertLatestFile(const stl_wrappers::KVMap& file_contents); private: uint32_t GetAndWriteNextID(WritableFile* file) const; diff --git a/table/table_test.cc b/table/table_test.cc index 444d9bd89..311361d09 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -13,15 +13,14 @@ #include #include #include -#include #include +#include #include #include "db/dbformat.h" #include "db/memtable.h" #include "db/write_batch_internal.h" #include "db/writebuffer.h" - #include "rocksdb/cache.h" #include "rocksdb/db.h" #include "rocksdb/env.h" @@ -29,27 +28,23 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/slice_transform.h" #include "rocksdb/statistics.h" - #include "table/block.h" #include "table/block_based_table_builder.h" #include "table/block_based_table_factory.h" #include "table/block_based_table_reader.h" #include "table/block_builder.h" #include "table/format.h" +#include "table/get_context.h" #include "table/meta_blocks.h" #include "table/plain_table_factory.h" -#include "table/get_context.h" - #include "util/compression.h" #include "util/random.h" +#include "util/scoped_arena_iterator.h" #include "util/statistics.h" +#include "util/stl_wrappers.h" #include "util/string_util.h" #include "util/testharness.h" #include "util/testutil.h" -#include "util/scoped_arena_iterator.h" - -using std::vector; -using std::string; namespace rocksdb { @@ -106,26 +101,14 @@ void Increment(const Comparator* cmp, std::string* key) { } } -// An STL comparator that uses a Comparator -struct STLLessThan { - const Comparator* cmp; - - STLLessThan() : cmp(BytewiseComparator()) { } - explicit STLLessThan(const Comparator* c) : cmp(c) { } - bool operator()(const std::string& a, const std::string& b) const { - return cmp->Compare(Slice(a), Slice(b)) < 0; - } -}; - } // namespace -typedef std::map KVMap; - // Helper class for tests to unify the interface between // BlockBuilder/TableBuilder and Block/Table. class Constructor { public: - explicit Constructor(const Comparator* cmp) : data_(STLLessThan(cmp)) {} + explicit Constructor(const Comparator* cmp) + : data_(stl_wrappers::LessOfComparator(cmp)) {} virtual ~Constructor() { } void Add(const std::string& key, const Slice& value) { @@ -135,18 +118,15 @@ class Constructor { // Finish constructing the data structure with all the keys that have // been added so far. Returns the keys in sorted order in "*keys" // and stores the key/value pairs in "*kvmap" - void Finish(const Options& options, - const ImmutableCFOptions& ioptions, + void Finish(const Options& options, const ImmutableCFOptions& ioptions, const BlockBasedTableOptions& table_options, const InternalKeyComparator& internal_comparator, - std::vector* keys, KVMap* kvmap) { + std::vector* keys, stl_wrappers::KVMap* kvmap) { last_internal_key_ = &internal_comparator; *kvmap = data_; keys->clear(); - for (KVMap::const_iterator it = data_.begin(); - it != data_.end(); - ++it) { - keys->push_back(it->first); + for (const auto& kv : data_) { + keys->push_back(kv.first); } data_.clear(); Status s = FinishImpl(options, ioptions, table_options, @@ -159,11 +139,11 @@ class Constructor { const ImmutableCFOptions& ioptions, const BlockBasedTableOptions& table_options, const InternalKeyComparator& internal_comparator, - const KVMap& data) = 0; + const stl_wrappers::KVMap& data) = 0; virtual Iterator* NewIterator() const = 0; - virtual const KVMap& data() { return data_; } + virtual const stl_wrappers::KVMap& data() { return data_; } virtual bool IsArenaMode() const { return false; } @@ -175,7 +155,7 @@ class Constructor { const InternalKeyComparator* last_internal_key_; private: - KVMap data_; + stl_wrappers::KVMap data_; }; class BlockConstructor: public Constructor { @@ -191,7 +171,7 @@ class BlockConstructor: public Constructor { const ImmutableCFOptions& ioptions, const BlockBasedTableOptions& table_options, const InternalKeyComparator& internal_comparator, - const KVMap& kv_map) override { + const stl_wrappers::KVMap& kv_map) override { delete block_; block_ = nullptr; BlockBuilder builder(table_options.block_restart_interval); @@ -280,7 +260,7 @@ class TableConstructor: public Constructor { const ImmutableCFOptions& ioptions, const BlockBasedTableOptions& table_options, const InternalKeyComparator& internal_comparator, - const KVMap& kv_map) override { + const stl_wrappers::KVMap& kv_map) override { Reset(); file_writer_.reset(test::GetWritableFileWriter(new test::StringSink())); unique_ptr builder; @@ -393,7 +373,7 @@ class MemTableConstructor: public Constructor { virtual Status FinishImpl(const Options&, const ImmutableCFOptions& ioptions, const BlockBasedTableOptions& table_options, const InternalKeyComparator& internal_comparator, - const KVMap& kv_map) override { + const stl_wrappers::KVMap& kv_map) override { delete memtable_->Unref(); ImmutableCFOptions mem_ioptions(ioptions); memtable_ = new MemTable(internal_comparator_, mem_ioptions, @@ -440,7 +420,7 @@ class DBConstructor: public Constructor { const ImmutableCFOptions& ioptions, const BlockBasedTableOptions& table_options, const InternalKeyComparator& internal_comparator, - const KVMap& kv_map) override { + const stl_wrappers::KVMap& kv_map) override { delete db_; db_ = nullptr; NewDB(); @@ -705,7 +685,7 @@ class HarnessTest : public testing::Test { void Test(Random* rnd) { std::vector keys; - KVMap data; + stl_wrappers::KVMap data; constructor_->Finish(options_, ioptions_, table_options_, *internal_comparator_, &keys, &data); @@ -717,13 +697,12 @@ class HarnessTest : public testing::Test { } void TestForwardScan(const std::vector& keys, - const KVMap& data) { + const stl_wrappers::KVMap& data) { Iterator* iter = constructor_->NewIterator(); ASSERT_TRUE(!iter->Valid()); iter->SeekToFirst(); - for (KVMap::const_iterator model_iter = data.begin(); - model_iter != data.end(); - ++model_iter) { + for (stl_wrappers::KVMap::const_iterator model_iter = data.begin(); + model_iter != data.end(); ++model_iter) { ASSERT_EQ(ToString(data, model_iter), ToString(iter)); iter->Next(); } @@ -736,13 +715,12 @@ class HarnessTest : public testing::Test { } void TestBackwardScan(const std::vector& keys, - const KVMap& data) { + const stl_wrappers::KVMap& data) { Iterator* iter = constructor_->NewIterator(); ASSERT_TRUE(!iter->Valid()); iter->SeekToLast(); - for (KVMap::const_reverse_iterator model_iter = data.rbegin(); - model_iter != data.rend(); - ++model_iter) { + for (stl_wrappers::KVMap::const_reverse_iterator model_iter = data.rbegin(); + model_iter != data.rend(); ++model_iter) { ASSERT_EQ(ToString(data, model_iter), ToString(iter)); iter->Prev(); } @@ -754,13 +732,12 @@ class HarnessTest : public testing::Test { } } - void TestRandomAccess(Random* rnd, - const std::vector& keys, - const KVMap& data) { + void TestRandomAccess(Random* rnd, const std::vector& keys, + const stl_wrappers::KVMap& data) { static const bool kVerbose = false; Iterator* iter = constructor_->NewIterator(); ASSERT_TRUE(!iter->Valid()); - KVMap::const_iterator model_iter = data.begin(); + stl_wrappers::KVMap::const_iterator model_iter = data.begin(); if (kVerbose) fprintf(stderr, "---\n"); for (int i = 0; i < 200; i++) { const int toss = rnd->Uniform(support_prev_ ? 5 : 3); @@ -828,7 +805,8 @@ class HarnessTest : public testing::Test { } } - std::string ToString(const KVMap& data, const KVMap::const_iterator& it) { + std::string ToString(const stl_wrappers::KVMap& data, + const stl_wrappers::KVMap::const_iterator& it) { if (it == data.end()) { return "END"; } else { @@ -836,8 +814,8 @@ class HarnessTest : public testing::Test { } } - std::string ToString(const KVMap& data, - const KVMap::const_reverse_iterator& it) { + std::string ToString(const stl_wrappers::KVMap& data, + const stl_wrappers::KVMap::const_reverse_iterator& it) { if (it == data.rend()) { return "END"; } else { @@ -982,7 +960,7 @@ TEST_F(BlockBasedTableTest, BasicBlockBasedTableProperties) { c.Add("j9", "val9"); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; Options options; options.compression = kNoCompression; BlockBasedTableOptions table_options; @@ -1017,7 +995,7 @@ TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) { TableConstructor c(BytewiseComparator(), true); c.Add("a1", "val1"); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; BlockBasedTableOptions table_options; table_options.filter_policy.reset(NewBloomFilterPolicy(10)); Options options; @@ -1034,8 +1012,8 @@ TEST_F(BlockBasedTableTest, FilterPolicyNameProperties) { // BlockBasedTableTest::PrefetchTest // void AssertKeysInCache(BlockBasedTable* table_reader, - const vector& keys_in_cache, - const vector& keys_not_in_cache) { + const std::vector& keys_in_cache, + const std::vector& keys_not_in_cache) { for (auto key : keys_in_cache) { ASSERT_TRUE(table_reader->TEST_KeyInCache(ReadOptions(), key)); } @@ -1047,10 +1025,10 @@ void AssertKeysInCache(BlockBasedTable* table_reader, void PrefetchRange(TableConstructor* c, Options* opt, BlockBasedTableOptions* table_options, - const vector& keys, - const char* key_begin, const char* key_end, - const vector& keys_in_cache, - const vector& keys_not_in_cache, + const std::vector& keys, const char* key_begin, + const char* key_end, + const std::vector& keys_in_cache, + const std::vector& keys_not_in_cache, const Status expected_status = Status::OK()) { // reset the cache and reopen the table table_options->block_cache = NewLRUCache(16 * 1024 * 1024); @@ -1093,7 +1071,7 @@ TEST_F(BlockBasedTableTest, PrefetchTest) { c.Add("k06", "hello3"); c.Add("k07", std::string(100000, 'x')); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; const ImmutableCFOptions ioptions(opt); c.Finish(opt, ioptions, table_options, *ikc, &keys, &kvmap); @@ -1197,7 +1175,7 @@ TEST_F(BlockBasedTableTest, TotalOrderSeekOnHashIndex) { c.Add("abbb1", std::string('a', 56)); c.Add("cccc2", std::string('a', 56)); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; const ImmutableCFOptions ioptions(options); c.Finish(options, ioptions, table_options, GetPlainInternalComparator(options.comparator), &keys, &kvmap); @@ -1271,7 +1249,7 @@ TEST_F(TableTest, HashIndexTest) { AddInternalKey(&c, "0095"); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; Options options; options.prefix_extractor.reset(NewFixedPrefixTransform(3)); BlockBasedTableOptions table_options; @@ -1385,7 +1363,7 @@ TEST_F(BlockBasedTableTest, IndexSizeStat) { } std::vector ks; - KVMap kvmap; + stl_wrappers::KVMap kvmap; Options options; options.compression = kNoCompression; BlockBasedTableOptions table_options; @@ -1418,7 +1396,7 @@ TEST_F(BlockBasedTableTest, NumBlockStat) { } std::vector ks; - KVMap kvmap; + stl_wrappers::KVMap kvmap; const ImmutableCFOptions ioptions(options); c.Finish(options, ioptions, table_options, GetPlainInternalComparator(options.comparator), &ks, &kvmap); @@ -1491,7 +1469,7 @@ TEST_F(BlockBasedTableTest, BlockCacheDisabledTest) { table_options.filter_policy.reset(NewBloomFilterPolicy(10)); options.table_factory.reset(new BlockBasedTableFactory(table_options)); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; TableConstructor c(BytewiseComparator(), true); c.Add("key", "value"); @@ -1537,7 +1515,7 @@ TEST_F(BlockBasedTableTest, FilterBlockInBlockCache) { table_options.cache_index_and_filter_blocks = true; options.table_factory.reset(new BlockBasedTableFactory(table_options)); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; TableConstructor c(BytewiseComparator()); c.Add("key", "value"); @@ -1681,7 +1659,7 @@ TEST_F(BlockBasedTableTest, BlockCacheLeak) { c.Add("k06", "hello3"); c.Add("k07", std::string(100000, 'x')); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; const ImmutableCFOptions ioptions(opt); c.Finish(opt, ioptions, table_options, *ikc, &keys, &kvmap); @@ -1775,7 +1753,7 @@ TEST_F(GeneralTableTest, ApproximateOffsetOfPlain) { c.Add("k06", "hello3"); c.Add("k07", std::string(100000, 'x')); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; Options options; test::PlainInternalKeyComparator internal_comparator(options.comparator); options.compression = kNoCompression; @@ -1807,7 +1785,7 @@ static void DoCompressionTest(CompressionType comp) { c.Add("k03", "hello3"); c.Add("k04", test::CompressibleString(&rnd, 0.25, 10000, &tmp)); std::vector keys; - KVMap kvmap; + stl_wrappers::KVMap kvmap; Options options; test::PlainInternalKeyComparator ikc(options.comparator); options.compression = comp; diff --git a/util/hash_cuckoo_rep.cc b/util/hash_cuckoo_rep.cc index 5f664847e..ea2b7bca2 100644 --- a/util/hash_cuckoo_rep.cc +++ b/util/hash_cuckoo_rep.cc @@ -5,20 +5,21 @@ // #ifndef ROCKSDB_LITE + #include "util/hash_cuckoo_rep.h" #include #include #include +#include #include #include -#include #include -#include "rocksdb/memtablerep.h" -#include "util/murmurhash.h" #include "db/memtable.h" #include "db/skiplist.h" +#include "rocksdb/memtablerep.h" +#include "util/murmurhash.h" #include "util/stl_wrappers.h" namespace rocksdb { @@ -40,7 +41,7 @@ struct CuckooStep { CuckooStep() : bucket_id_(-1), prev_step_id_(kNullStep), depth_(1) {} // MSVC does not support = default yet - CuckooStep(CuckooStep&& o) { *this = std::move(o); } + CuckooStep(CuckooStep&& o) ROCKSDB_NOEXCEPT { *this = std::move(o); } CuckooStep& operator=(CuckooStep&& rhs) { bucket_id_ = std::move(rhs.bucket_id_); diff --git a/util/stl_wrappers.h b/util/stl_wrappers.h index b4c14b4ba..15b9bdf52 100644 --- a/util/stl_wrappers.h +++ b/util/stl_wrappers.h @@ -2,31 +2,45 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -// #pragma once -#include "util/murmurhash.h" -#include "util/coding.h" +#include +#include +#include "rocksdb/comparator.h" #include "rocksdb/memtablerep.h" #include "rocksdb/slice.h" +#include "util/coding.h" +#include "util/murmurhash.h" namespace rocksdb { namespace stl_wrappers { - class Base { - protected: - const MemTableRep::KeyComparator& compare_; - explicit Base(const MemTableRep::KeyComparator& compare) - : compare_(compare) { } - }; - struct Compare : private Base { - explicit Compare(const MemTableRep::KeyComparator& compare) - : Base(compare) { } - inline bool operator()(const char* a, const char* b) const { - return compare_(a, b) < 0; - } - }; +class Base { + protected: + const MemTableRep::KeyComparator& compare_; + explicit Base(const MemTableRep::KeyComparator& compare) + : compare_(compare) {} +}; +struct Compare : private Base { + explicit Compare(const MemTableRep::KeyComparator& compare) : Base(compare) {} + inline bool operator()(const char* a, const char* b) const { + return compare_(a, b) < 0; + } +}; + +struct LessOfComparator { + explicit LessOfComparator(const Comparator* c = BytewiseComparator()) + : cmp(c) {} + + bool operator()(const std::string& a, const std::string& b) const { + return cmp->Compare(Slice(a), Slice(b)) < 0; + } + + const Comparator* cmp; +}; + +typedef std::map KVMap; } } diff --git a/utilities/write_batch_with_index/write_batch_with_index.cc b/utilities/write_batch_with_index/write_batch_with_index.cc index 94b227dc9..c4a4a1500 100644 --- a/utilities/write_batch_with_index/write_batch_with_index.cc +++ b/utilities/write_batch_with_index/write_batch_with_index.cc @@ -7,15 +7,15 @@ #include "rocksdb/utilities/write_batch_with_index.h" -#include #include +#include -#include "rocksdb/comparator.h" -#include "rocksdb/iterator.h" #include "db/column_family.h" #include "db/merge_context.h" #include "db/merge_helper.h" #include "db/skiplist.h" +#include "rocksdb/comparator.h" +#include "rocksdb/iterator.h" #include "util/arena.h" #include "utilities/write_batch_with_index/write_batch_with_index_internal.h" diff --git a/utilities/write_batch_with_index/write_batch_with_index_internal.h b/utilities/write_batch_with_index/write_batch_with_index_internal.h index 38c5b217d..54bbd81da 100644 --- a/utilities/write_batch_with_index/write_batch_with_index_internal.h +++ b/utilities/write_batch_with_index/write_batch_with_index_internal.h @@ -2,11 +2,10 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. +#pragma once #ifndef ROCKSDB_LITE -#pragma once - #include #include #include