andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make MemTableRepFactory into a Customizable class (#8419)

Browse Source 
Summary:
This PR does the following:
-> Makes the MemTableRepFactory into a Customizable class and creatable/configurable via CreateFromString
-> Makes the existing implementations compatible with configurations
-> Moves the "SpecialRepFactory" test class into testutil, accessible via the ObjectRegistry or a NewSpecial API

New tests were added to validate the functionality and all existing tests pass.  db_bench and memtablerep_bench were hand-tested to verify the functionality in those tools.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8419

Reviewed By: zhichao-cao

Differential Revision: D29558961

Pulled By: mrambacher

fbshipit-source-id: 81b7229636e4e649a0c914e73ac7b0f8454c931c
This commit is contained in:
mrambacher 2021-09-08 07:45:59 -07:00 committed by Facebook GitHub Bot
parent e40b04e9fa
commit beed86473a
34 changed files with 767 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/column_family.cc
View File

@ -29,7 +29,6 @@
#include "db/version_set.h"
#include "db/write_controller.h"
#include "file/sst_file_manager_impl.h"
#include "memtable/hash_skiplist_rep.h"
#include "monitoring/thread_status_util.h"
#include "options/options_helper.h"
#include "port/port.h"

12
db/column_family_test.cc
View File

@ -2975,7 +2975,8 @@ TEST_P(ColumnFamilyTest, FlushCloseWALFiles) {
SpecialEnv env(Env::Default());
db_options_.env = &env;
db_options_.max_background_flushes = 1;
column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2));
column_family_options_.memtable_factory.reset(
test::NewSpecialSkipListFactory(2));
Open();
CreateColumnFamilies({"one"});
ASSERT_OK(Put(1, "fodor", "mirko"));
@ -3020,7 +3021,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) {
SpecialEnv env(Env::Default());
db_options_.env = &env;
db_options_.max_background_flushes = 1;
column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2));
column_family_options_.memtable_factory.reset(
test::NewSpecialSkipListFactory(2));
Open();
CreateColumnFamilies({"one"});
ASSERT_OK(Put(1, "fodor", "mirko"));
@ -3071,7 +3073,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile2) {
env.SetBackgroundThreads(2, Env::HIGH);
db_options_.env = &env;
db_options_.max_background_flushes = 1;
column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2));
column_family_options_.memtable_factory.reset(
test::NewSpecialSkipListFactory(2));
Open();
CreateColumnFamilies({"one"});
ASSERT_OK(Put(1, "fodor", "mirko"));
@ -3129,7 +3132,8 @@ TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
env.SetBackgroundThreads(2, Env::HIGH);
db_options_.env = &env;
db_options_.max_background_flushes = 1;
column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(3));
column_family_options_.memtable_factory.reset(
test::NewSpecialSkipListFactory(3));
column_family_options_.level0_file_num_compaction_trigger = 2;
Open();
CreateColumnFamilies({"one"});

1
db/compaction/compaction_job_stats_test.cc
View File

@ -24,7 +24,6 @@
#include "db/write_batch_internal.h"
#include "env/mock_env.h"
#include "file/filename.h"
#include "memtable/hash_linklist_rep.h"
#include "monitoring/statistics.h"
#include "monitoring/thread_status_util.h"
#include "port/stack_trace.h"

4
db/db_bloom_filter_test.cc
View File

@ -15,6 +15,7 @@
#include "port/stack_trace.h"
#include "rocksdb/perf_context.h"
#include "table/block_based/filter_policy_internal.h"
#include "test_util/testutil.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
@ -2171,7 +2172,8 @@ TEST_F(DBBloomFilterTest, SeekForPrevWithPartitionedFilters) {
Options options = CurrentOptions();
constexpr size_t kNumKeys = 10000;
static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000");
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeys + 10));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeys + 10));
options.create_if_missing = true;
constexpr size_t kPrefixLength = 4;
options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength));

25
db/db_compaction_test.cc
View File

@ -19,6 +19,7 @@
#include "rocksdb/sst_file_writer.h"
#include "rocksdb/utilities/convenience.h"
#include "test_util/sync_point.h"
#include "test_util/testutil.h"
#include "util/concurrent_task_limiter_impl.h"
#include "util/random.h"
#include "utilities/fault_injection_env.h"
@ -805,7 +806,8 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
options.num_levels = 3;
options.level0_file_num_compaction_trigger = 3;
options.max_subcompactions = max_subcompactions_;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
CreateAndReopenWithCF({"pikachu"}, options);
Random rnd(301);
@ -852,7 +854,8 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
options.level0_slowdown_writes_trigger = 20;
options.soft_pending_compaction_bytes_limit = 1 << 30; // Infinitely large
options.max_background_compactions = 3;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
// Block all threads in thread pool.
const size_t kTotalTasks = 4;
@ -2094,7 +2097,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) {
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
@ -2203,7 +2206,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
@ -2313,7 +2316,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionCFPathUse) {
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
@ -3012,7 +3015,7 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
}
Options options = CurrentOptions();
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
@ -4815,7 +4818,8 @@ TEST_F(DBCompactionTest, CompactionLimiter) {
options.level0_slowdown_writes_trigger = 64;
options.level0_stop_writes_trigger = 64;
options.max_background_jobs = kMaxBackgroundThreads; // Enough threads
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.max_write_buffer_number = 10; // Enough memtables
DestroyAndReopen(options);
@ -5338,7 +5342,8 @@ TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) {
options.level0_file_num_compaction_trigger =
options.level0_stop_writes_trigger;
options.max_subcompactions = max_subcompactions_;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
Random rnd(301);
@ -5864,7 +5869,7 @@ TEST_F(DBCompactionTest, ChangeLevelCompactRangeConflictsWithManual) {
// `Status::Incomplete`.
Options options = CurrentOptions();
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 3;
Reopen(options);
@ -5952,7 +5957,7 @@ TEST_F(DBCompactionTest, ChangeLevelErrorPathTest) {
// succeeds
Options options = CurrentOptions();
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 3;
Reopen(options);

5
db/db_flush_test.cc
View File

@ -18,6 +18,7 @@
#include "port/stack_trace.h"
#include "rocksdb/utilities/transaction_db.h"
#include "test_util/sync_point.h"
#include "test_util/testutil.h"
#include "util/cast_util.h"
#include "util/mutexlock.h"
#include "utilities/fault_injection_env.h"
@ -139,7 +140,7 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
// scheduled in the low-pri (compaction) thread pool.
Options options = CurrentOptions();
options.level0_file_num_compaction_trigger = 4;
options.memtable_factory.reset(new SpecialSkipListFactory(1));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
Reopen(options);
env_->SetBackgroundThreads(0, Env::HIGH);
@ -2320,7 +2321,7 @@ TEST_P(DBAtomicFlushTest, TriggerFlushAndClose) {
options.create_if_missing = true;
options.atomic_flush = atomic_flush;
options.memtable_factory.reset(
new SpecialSkipListFactory(kNumKeysTriggerFlush));
test::NewSpecialSkipListFactory(kNumKeysTriggerFlush));
CreateAndReopenWithCF({"pikachu"}, options);
for (int i = 0; i != kNumKeysTriggerFlush; ++i) {

2
db/db_impl/db_impl.cc
View File

@ -61,8 +61,6 @@
#include "logging/auto_roll_logger.h"
#include "logging/log_buffer.h"
#include "logging/logging.h"
#include "memtable/hash_linklist_rep.h"
#include "memtable/hash_skiplist_rep.h"
#include "monitoring/in_memory_stats_history.h"
#include "monitoring/instrumented_mutex.h"
#include "monitoring/iostats_context_imp.h"

7
db/db_io_failure_test.cc
View File

@ -9,6 +9,7 @@
#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "test_util/testutil.h"
#include "util/random.h"
namespace ROCKSDB_NAMESPACE {
@ -269,7 +270,7 @@ TEST_F(DBIOFailureTest, FlushSstRangeSyncError) {
options.writable_file_max_buffer_size = 128 * 1024;
options.bytes_per_sync = 128 * 1024;
options.level0_file_num_compaction_trigger = 4;
options.memtable_factory.reset(new SpecialSkipListFactory(10));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(10));
BlockBasedTableOptions table_options;
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
@ -396,7 +397,7 @@ TEST_F(DBIOFailureTest, FlushSstCloseError) {
options.error_if_exists = false;
options.paranoid_checks = true;
options.level0_file_num_compaction_trigger = 4;
options.memtable_factory.reset(new SpecialSkipListFactory(2));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(2));
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
@ -492,7 +493,7 @@ TEST_F(DBIOFailureTest, FlushSstSyncError) {
options.paranoid_checks = true;
options.use_fsync = false;
options.level0_file_num_compaction_trigger = 4;
options.memtable_factory.reset(new SpecialSkipListFactory(2));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(2));
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);

28
db/db_range_del_test.cc
View File

@ -118,7 +118,7 @@ TEST_F(DBRangeDelTest, CompactionOutputFilesExactlyFilled) {
Options options = CurrentOptions();
options.disable_auto_compactions = true;
options.level0_file_num_compaction_trigger = kNumFiles;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
options.num_levels = 2;
options.target_file_size_base = kFileBytes;
BlockBasedTableOptions table_options;
@ -166,7 +166,7 @@ TEST_F(DBRangeDelTest, MaxCompactionBytesCutsOutputFiles) {
opts.disable_auto_compactions = true;
opts.level0_file_num_compaction_trigger = kNumFiles;
opts.max_compaction_bytes = kNumPerFile * kBytesPerVal;
opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
// Want max_compaction_bytes to trigger the end of compaction output file, not
// target_file_size_base, so make the latter much bigger
opts.target_file_size_base = 100 * opts.max_compaction_bytes;
@ -319,7 +319,7 @@ TEST_F(DBRangeDelTest, CompactionRemovesCoveredKeys) {
Options opts = CurrentOptions();
opts.comparator = test::Uint64Comparator();
opts.disable_auto_compactions = true;
opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
opts.num_levels = 2;
opts.statistics = CreateDBStatistics();
DestroyAndReopen(opts);
@ -377,7 +377,7 @@ TEST_F(DBRangeDelTest, ValidLevelSubcompactionBoundaries) {
options.level0_file_num_compaction_trigger = kNumFiles;
options.max_bytes_for_level_base = 2 * kFileBytes;
options.max_subcompactions = 4;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
options.num_levels = 3;
options.target_file_size_base = kFileBytes;
options.target_file_size_multiplier = 1;
@ -436,7 +436,7 @@ TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) {
options.compaction_style = kCompactionStyleUniversal;
options.level0_file_num_compaction_trigger = kFilesPerLevel;
options.max_subcompactions = 4;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
options.num_levels = kNumLevels;
options.target_file_size_base = kNumPerFile << 10;
options.target_file_size_multiplier = 1;
@ -489,7 +489,7 @@ TEST_F(DBRangeDelTest, CompactionRemovesCoveredMergeOperands) {
const int kNumPerFile = 3, kNumFiles = 3;
Options opts = CurrentOptions();
opts.disable_auto_compactions = true;
opts.memtable_factory.reset(new SpecialSkipListFactory(2 * kNumPerFile));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(2 * kNumPerFile));
opts.merge_operator = MergeOperators::CreateUInt64AddOperator();
opts.num_levels = 2;
Reopen(opts);
@ -613,7 +613,7 @@ TEST_F(DBRangeDelTest, TableEvictedDuringScan) {
opts.comparator = test::Uint64Comparator();
opts.level0_file_num_compaction_trigger = 4;
opts.level0_stop_writes_trigger = 4;
opts.memtable_factory.reset(new SpecialSkipListFactory(1));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
opts.num_levels = 2;
BlockBasedTableOptions bbto;
bbto.cache_index_and_filter_blocks = true;
@ -694,7 +694,7 @@ TEST_F(DBRangeDelTest, GetCoveredKeyFromImmutableMemtable) {
// memtable can hold. It switches the active memtable to immutable (flush is
// prevented by the above options) upon inserting an element that would
// overflow the memtable.
opts.memtable_factory.reset(new SpecialSkipListFactory(1));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
DestroyAndReopen(opts);
ASSERT_OK(db_->Put(WriteOptions(), "key", "val"));
@ -759,7 +759,7 @@ TEST_F(DBRangeDelTest, GetIgnoresRangeDeletions) {
Options opts = CurrentOptions();
opts.max_write_buffer_number = 4;
opts.min_write_buffer_number_to_merge = 3;
opts.memtable_factory.reset(new SpecialSkipListFactory(1));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
Reopen(opts);
ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val"));
@ -788,7 +788,7 @@ TEST_F(DBRangeDelTest, IteratorRemovesCoveredKeys) {
const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25;
Options opts = CurrentOptions();
opts.comparator = test::Uint64Comparator();
opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
DestroyAndReopen(opts);
// Write half of the keys before the tombstone and half after the tombstone.
@ -823,7 +823,7 @@ TEST_F(DBRangeDelTest, IteratorOverUserSnapshot) {
const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25;
Options opts = CurrentOptions();
opts.comparator = test::Uint64Comparator();
opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
DestroyAndReopen(opts);
const Snapshot* snapshot = nullptr;
@ -857,7 +857,7 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) {
Options opts = CurrentOptions();
opts.max_write_buffer_number = 4;
opts.min_write_buffer_number_to_merge = 3;
opts.memtable_factory.reset(new SpecialSkipListFactory(1));
opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
Reopen(opts);
ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val"));
@ -1005,7 +1005,7 @@ TEST_F(DBRangeDelTest, CompactionTreatsSplitInputLevelDeletionAtomically) {
options.compression = kNoCompression;
options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
options.memtable_factory.reset(
new SpecialSkipListFactory(2 /* num_entries_flush */));
test::NewSpecialSkipListFactory(2 /* num_entries_flush */));
options.target_file_size_base = kValueBytes;
// i == 0: CompactFiles
// i == 1: CompactRange
@ -1073,7 +1073,7 @@ TEST_F(DBRangeDelTest, RangeTombstoneEndKeyAsSstableUpperBound) {
options.compression = kNoCompression;
options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
options.memtable_factory.reset(
new SpecialSkipListFactory(2 /* num_entries_flush */));
test::NewSpecialSkipListFactory(2 /* num_entries_flush */));
options.target_file_size_base = kValueBytes;
options.disable_auto_compactions = true;

7
db/db_secondary_test.cc
View File

@ -12,6 +12,7 @@
#include "port/stack_trace.h"
#include "rocksdb/utilities/transaction_db.h"
#include "test_util/sync_point.h"
#include "test_util/testutil.h"
#include "utilities/fault_injection_env.h"
namespace ROCKSDB_NAMESPACE {
@ -937,7 +938,7 @@ TEST_F(DBSecondaryTest, DISABLED_SwitchWAL) {
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 2;
options.memtable_factory.reset(
new SpecialSkipListFactory(kNumKeysPerMemtable));
test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
Reopen(options);
Options options1;
@ -992,7 +993,7 @@ TEST_F(DBSecondaryTest, DISABLED_SwitchWALMultiColumnFamilies) {
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 2;
options.memtable_factory.reset(
new SpecialSkipListFactory(kNumKeysPerMemtable));
test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
CreateAndReopenWithCF({kCFName1}, options);
Options options1;
@ -1056,7 +1057,7 @@ TEST_F(DBSecondaryTest, CatchUpAfterFlush) {
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 2;
options.memtable_factory.reset(
new SpecialSkipListFactory(kNumKeysPerMemtable));
test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
Reopen(options);
Options options1;

11
db/db_test.cc
View File

@ -34,7 +34,6 @@
#include "db/write_batch_internal.h"
#include "env/mock_env.h"
#include "file/filename.h"
#include "memtable/hash_linklist_rep.h"
#include "monitoring/thread_status_util.h"
#include "port/port.h"
#include "port/stack_trace.h"
@ -5732,8 +5731,8 @@ TEST_F(DBTest, DISABLED_SuggestCompactRangeTest) {
};
Options options = CurrentOptions();
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(
DBTestBase::kNumKeysByGenerateNewRandomFile));
options.compaction_style = kCompactionStyleLevel;
options.compaction_filter_factory.reset(
new CompactionFilterFactoryGetContext());
@ -6138,7 +6137,7 @@ TEST_F(DBTest, DelayedWriteRate) {
options.level0_stop_writes_trigger = 999999;
options.delayed_write_rate = 20000000; // Start with 200MB/s
options.memtable_factory.reset(
new SpecialSkipListFactory(kEntriesPerMemTable));
test::NewSpecialSkipListFactory(kEntriesPerMemTable));
SetTimeElapseOnlySleepOnReopen(&options);
CreateAndReopenWithCF({"pikachu"}, options);
@ -6201,7 +6200,7 @@ TEST_F(DBTest, HardLimit) {
options.max_bytes_for_level_base = 10000000000u;
options.max_background_compactions = 1;
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
env_->SetBackgroundThreads(1, Env::LOW);
test::SleepingBackgroundTask sleeping_task_low;
@ -6450,7 +6449,7 @@ TEST_F(DBTest, LastWriteBufferDelay) {
options.disable_auto_compactions = true;
int kNumKeysPerMemtable = 3;
options.memtable_factory.reset(
new SpecialSkipListFactory(kNumKeysPerMemtable));
test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
Reopen(options);
test::SleepingBackgroundTask sleeping_task;

12
db/db_test2.cc
View File

@ -22,6 +22,7 @@
#include "rocksdb/trace_record_result.h"
#include "rocksdb/utilities/replayer.h"
#include "rocksdb/wal_filter.h"
#include "test_util/testutil.h"
#include "util/random.h"
#include "utilities/fault_injection_env.h"
@ -1264,7 +1265,7 @@ TEST_F(DBTest2, PresetCompressionDict) {
options.disable_auto_compactions = true;
options.level0_file_num_compaction_trigger = kNumL0Files;
options.memtable_factory.reset(
new SpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes));
test::NewSpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes));
options.num_levels = 2;
options.target_file_size_base = kL0FileBytes;
options.target_file_size_multiplier = 2;
@ -1484,7 +1485,7 @@ TEST_P(PresetCompressionDictTest, Flush) {
options.compression_opts.max_dict_bytes = kDictLen;
options.compression_opts.max_dict_buffer_bytes = kBlockLen;
}
options.memtable_factory.reset(new SpecialSkipListFactory(kKeysPerFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(kKeysPerFile));
options.statistics = CreateDBStatistics();
BlockBasedTableOptions bbto;
bbto.block_size = kBlockLen;
@ -2319,8 +2320,8 @@ INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
#ifndef ROCKSDB_LITE
TEST_F(DBTest2, MaxCompactionBytesTest) {
Options options = CurrentOptions();
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(
DBTestBase::kNumKeysByGenerateNewRandomFile));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 200 << 10;
options.arena_block_size = 4 << 10;
@ -3945,7 +3946,8 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
Options options = CurrentOptions();
options.compression = kNoCompression;
options.level0_file_num_compaction_trigger = kNumL0Files;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.new_table_reader_for_compaction_inputs = true;
// takes roughly one second, split into 100 x 10ms intervals. Each interval
// permits 5.12KB, which is smaller than the block size, so this test

89
db/db_test_util.h
View File

@ -24,7 +24,6 @@
#include "db/db_impl/db_impl.h"
#include "db/dbformat.h"
#include "file/filename.h"
#include "memtable/hash_linklist_rep.h"
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/convenience.h"
@ -112,94 +111,6 @@ struct OptionsOverride {
enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 };
// A hacky skip list mem table that triggers flush after number of entries.
class SpecialMemTableRep : public MemTableRep {
public:
explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
int num_entries_flush)
: MemTableRep(allocator),
memtable_(memtable),
num_entries_flush_(num_entries_flush),
num_entries_(0) {}
virtual KeyHandle Allocate(const size_t len, char** buf) override {
return memtable_->Allocate(len, buf);
}
// Insert key into the list.
// REQUIRES: nothing that compares equal to key is currently in the list.
virtual void Insert(KeyHandle handle) override {
num_entries_++;
memtable_->Insert(handle);
}
void InsertConcurrently(KeyHandle handle) override {
num_entries_++;
memtable_->Insert(handle);
}
// Returns true iff an entry that compares equal to key is in the list.
virtual bool Contains(const char* key) const override {
return memtable_->Contains(key);
}
virtual size_t ApproximateMemoryUsage() override {
// Return a high memory usage when number of entries exceeds the threshold
// to trigger a flush.
return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024;
}
virtual void Get(const LookupKey& k, void* callback_args,
bool (*callback_func)(void* arg,
const char* entry)) override {
memtable_->Get(k, callback_args, callback_func);
}
uint64_t ApproximateNumEntries(const Slice& start_ikey,
const Slice& end_ikey) override {
return memtable_->ApproximateNumEntries(start_ikey, end_ikey);
}
virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override {
return memtable_->GetIterator(arena);
}
virtual ~SpecialMemTableRep() override {}
private:
std::unique_ptr<MemTableRep> memtable_;
int num_entries_flush_;
int num_entries_;
};
// The factory for the hacky skip list mem table that triggers flush after
// number of entries exceeds a threshold.
class SpecialSkipListFactory : public MemTableRepFactory {
public:
// After number of inserts exceeds `num_entries_flush` in a mem table, trigger
// flush.
explicit SpecialSkipListFactory(int num_entries_flush)
: num_entries_flush_(num_entries_flush) {}
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* /*logger*/) override {
return new SpecialMemTableRep(
allocator, factory_.CreateMemTableRep(compare, allocator, transform, 0),
num_entries_flush_);
}
virtual const char* Name() const override { return "SkipListFactory"; }
bool IsInsertConcurrentlySupported() const override {
return factory_.IsInsertConcurrentlySupported();
}
private:
SkipListFactory factory_;
int num_entries_flush_;
};
// Special Env used to delay background operations
class SpecialEnv : public EnvWrapper {
public:

14
db/db_universal_compaction_test.cc
View File

@ -12,6 +12,7 @@
#if !defined(ROCKSDB_LITE)
#include "rocksdb/utilities/table_properties_collectors.h"
#include "test_util/sync_point.h"
#include "test_util/testutil.h"
#include "util/random.h"
namespace ROCKSDB_NAMESPACE {
@ -151,7 +152,7 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.optimize_filters_for_hits = true;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
options.memtable_factory.reset(new SpecialSkipListFactory(3));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(3));
DestroyAndReopen(options);
@ -1230,7 +1231,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 5;
options.write_buffer_size = 111 << 10; // 114KB
@ -1334,7 +1335,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCFPathUse) {
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 10;
options.write_buffer_size = 111 << 10; // 114KB
@ -1498,7 +1499,8 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
options.num_levels = 1;
options.write_buffer_size = 200 << 10; // 200KB
options.level0_file_num_compaction_trigger = 3;
options.memtable_factory.reset(new SpecialSkipListFactory(KNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(KNumKeysPerFile));
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options);
@ -1579,7 +1581,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 1;
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
std::vector<std::string> filenames;
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) {
@ -1729,7 +1731,7 @@ TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) {
const int kNumFilesTrigger = 8;
Options options = CurrentOptions();
options.memtable_factory.reset(
new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
options.compaction_options_universal.max_size_amplification_percent =
static_cast<unsigned int>(-1);

28
db/db_with_timestamp_basic_test.cc
View File

@ -16,6 +16,7 @@
#if !defined(ROCKSDB_LITE)
#include "test_util/sync_point.h"
#endif
#include "test_util/testutil.h"
#include "utilities/fault_injection_env.h"
namespace ROCKSDB_NAMESPACE {
@ -415,7 +416,8 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) {
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
const std::vector<uint64_t> start_keys = {1, 0};
const std::vector<std::string> write_timestamps = {Timestamp(1, 0),
@ -842,7 +844,8 @@ TEST_F(DBBasicTestWithTimestamp, SimpleForwardIterateLowerTsBound) {
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
const std::vector<std::string> write_timestamps = {Timestamp(1, 0),
Timestamp(3, 0)};
@ -927,7 +930,8 @@ TEST_F(DBBasicTestWithTimestamp, ForwardIterateStartSeqnum) {
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
std::vector<SequenceNumber> start_seqs;
@ -2235,7 +2239,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGet) {
Options options = CurrentOptions();
options.create_if_missing = true;
options.env = env_;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
size_t ts_sz = Timestamp(0, 0).size();
TestComparator test_cmp(ts_sz);
options.comparator = &test_cmp;
@ -2313,7 +2318,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutDeleteGet) {
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
const int kNumKeysPerFile = 1024;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
BlockBasedTableOptions bbto;
bbto.filter_policy = std::get<0>(GetParam());
bbto.whole_key_filtering = true;
@ -2442,7 +2448,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGetWithCompaction) {
Options options = CurrentOptions();
options.create_if_missing = true;
options.env = env_;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
FlushedFileCollector* collector = new FlushedFileCollector();
options.listeners.emplace_back(collector);
@ -2558,7 +2565,8 @@ TEST_F(DBBasicTestWithTimestamp, BatchWriteAndMultiGet) {
Options options = CurrentOptions();
options.create_if_missing = true;
options.env = env_;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.memtable_prefix_bloom_size_ratio = 0.1;
options.memtable_whole_key_filtering = true;
@ -2725,7 +2733,8 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, IterateWithPrefix) {
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.prefix_extractor = std::get<0>(GetParam());
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
BlockBasedTableOptions bbto;
bbto.filter_policy = std::get<1>(GetParam());
bbto.index_type = std::get<3>(GetParam());
@ -2879,7 +2888,8 @@ TEST_P(DBBasicTestWithTsIterTombstones, IterWithDelete) {
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.prefix_extractor = std::get<0>(GetParam());
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
BlockBasedTableOptions bbto;
bbto.filter_policy = std::get<1>(GetParam());
bbto.index_type = std::get<3>(GetParam());

4
db/db_with_timestamp_compaction_test.cc
View File

@ -10,6 +10,7 @@
#include "db/compaction/compaction.h"
#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "test_util/testutil.h"
namespace ROCKSDB_NAMESPACE {
@ -56,7 +57,8 @@ TEST_F(TimestampCompatibleCompactionTest, UserKeyCrossFileBoundary) {
options.comparator = test::ComparatorWithU64Ts();
options.level0_file_num_compaction_trigger = 3;
constexpr size_t kNumKeysPerFile = 101;
options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
options.memtable_factory.reset(
test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->ClearAllCallBacks();

17
db/listener_test.cc
View File

@ -10,7 +10,6 @@
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "file/filename.h"
#include "memtable/hash_linklist_rep.h"
#include "monitoring/statistics.h"
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
@ -529,8 +528,8 @@ TEST_F(EventListenerTest, CompactionReasonLevel) {
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(
DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
@ -595,8 +594,8 @@ TEST_F(EventListenerTest, CompactionReasonUniversal) {
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(
DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
@ -657,8 +656,8 @@ TEST_F(EventListenerTest, CompactionReasonFIFO) {
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
options.memtable_factory.reset(
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(
DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
@ -925,7 +924,7 @@ TEST_F(EventListenerTest, BackgroundErrorListenerFailedFlushTest) {
options.create_if_missing = true;
options.env = env_;
options.listeners.push_back(listener);
options.memtable_factory.reset(new SpecialSkipListFactory(1));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
options.paranoid_checks = true;
DestroyAndReopen(options);
@ -956,7 +955,7 @@ TEST_F(EventListenerTest, BackgroundErrorListenerFailedCompactionTest) {
options.env = env_;
options.level0_file_num_compaction_trigger = 2;
options.listeners.push_back(listener);
options.memtable_factory.reset(new SpecialSkipListFactory(2));
options.memtable_factory.reset(test::NewSpecialSkipListFactory(2));
options.paranoid_checks = true;
DestroyAndReopen(options);

35
include/rocksdb/memtablerep.h
View File

@ -42,6 +42,7 @@
#include <stdexcept>
#include <unordered_set>
#include "rocksdb/customizable.h"
#include "rocksdb/slice.h"
namespace ROCKSDB_NAMESPACE {
@ -51,6 +52,7 @@ class Allocator;
class LookupKey;
class SliceTransform;
class Logger;
struct DBOptions;
using KeyHandle = void*;
@ -290,10 +292,15 @@ class MemTableRep {
// This is the base class for all factories that are used by RocksDB to create
// new MemTableRep objects
class MemTableRepFactory {
class MemTableRepFactory : public Customizable {
public:
virtual ~MemTableRepFactory() {}
static const char* Type() { return "MemTableRepFactory"; }
static Status CreateFromString(const ConfigOptions& config_options,
const std::string& id,
std::unique_ptr<MemTableRepFactory>* factory);
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Allocator*, const SliceTransform*,
Logger* logger) = 0;
@ -326,20 +333,27 @@ class MemTableRepFactory {
// seeks with consecutive keys.
class SkipListFactory : public MemTableRepFactory {
public:
explicit SkipListFactory(size_t lookahead = 0) : lookahead_(lookahead) {}
explicit SkipListFactory(size_t lookahead = 0);
// Methods for Configurable/Customizable class overrides
static const char* kClassName() { return "SkipListFactory"; }
static const char* kNickName() { return "skip_list"; }
virtual const char* Name() const override { return kClassName(); }
virtual const char* NickName() const override { return kNickName(); }
std::string GetId() const override;
// Methods for MemTableRepFactory class overrides
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Allocator*, const SliceTransform*,
Logger* logger) override;
virtual const char* Name() const override { return "SkipListFactory"; }
bool IsInsertConcurrentlySupported() const override { return true; }
bool CanHandleDuplicatedKey() const override { return true; }
private:
const size_t lookahead_;
size_t lookahead_;
};
#ifndef ROCKSDB_LITE
@ -352,17 +366,22 @@ class SkipListFactory : public MemTableRepFactory {
// VectorRep. On initialization, the underlying array will be at least count
// bytes reserved for usage.
class VectorRepFactory : public MemTableRepFactory {
const size_t count_;
size_t count_;
public:
explicit VectorRepFactory(size_t count = 0) : count_(count) {}
explicit VectorRepFactory(size_t count = 0);
// Methods for Configurable/Customizable class overrides
static const char* kClassName() { return "VectorRepFactory"; }
static const char* kNickName() { return "vector"; }
const char* Name() const override { return kClassName(); }
const char* NickName() const override { return kNickName(); }
// Methods for MemTableRepFactory class overrides
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Allocator*, const SliceTransform*,
Logger* logger) override;
virtual const char* Name() const override { return "VectorRepFactory"; }
};
// This class contains a fixed array of buckets, each

1
include/rocksdb/utilities/options_type.h
View File

@ -36,7 +36,6 @@ enum class OptionType {
kSliceTransform,
kCompressionType,
kCompactionStopStyle,
kMemTableRepFactory,
kFilterPolicy,
kChecksumType,
kEncodingType,

75
memtable/hash_linklist_rep.cc
View File

@ -5,10 +5,10 @@
//
#ifndef ROCKSDB_LITE
#include "memtable/hash_linklist_rep.h"
#include <algorithm>
#include <atomic>
#include "db/memtable.h"
#include "memory/arena.h"
#include "memtable/skiplist.h"
@ -17,6 +17,7 @@
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/utilities/options_type.h"
#include "util/hash.h"
namespace ROCKSDB_NAMESPACE {
@ -820,15 +821,77 @@ Node* HashLinkListRep::FindGreaterOrEqualInBucket(Node* head,
return x;
}
} // anon namespace
struct HashLinkListRepOptions {
static const char* kName() { return "HashLinkListRepFactoryOptions"; }
size_t bucket_count;
uint32_t threshold_use_skiplist;
size_t huge_page_tlb_size;
int bucket_entries_logging_threshold;
bool if_log_bucket_dist_when_flash;
};
static std::unordered_map<std::string, OptionTypeInfo> hash_linklist_info = {
{"bucket_count",
{offsetof(struct HashLinkListRepOptions, bucket_count), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"threshold",
{offsetof(struct HashLinkListRepOptions, threshold_use_skiplist),
OptionType::kUInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"huge_page_size",
{offsetof(struct HashLinkListRepOptions, huge_page_tlb_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"logging_threshold",
{offsetof(struct HashLinkListRepOptions, bucket_entries_logging_threshold),
OptionType::kInt, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"log_when_flash",
{offsetof(struct HashLinkListRepOptions, if_log_bucket_dist_when_flash),
OptionType::kBoolean, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
};
class HashLinkListRepFactory : public MemTableRepFactory {
public:
explicit HashLinkListRepFactory(size_t bucket_count,
uint32_t threshold_use_skiplist,
size_t huge_page_tlb_size,
int bucket_entries_logging_threshold,
bool if_log_bucket_dist_when_flash) {
options_.bucket_count = bucket_count;
options_.threshold_use_skiplist = threshold_use_skiplist;
options_.huge_page_tlb_size = huge_page_tlb_size;
options_.bucket_entries_logging_threshold =
bucket_entries_logging_threshold;
options_.if_log_bucket_dist_when_flash = if_log_bucket_dist_when_flash;
RegisterOptions(&options_, &hash_linklist_info);
}
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* logger) override;
static const char* kClassName() { return "HashLinkListRepFactory"; }
static const char* kNickName() { return "hash_linkedlist"; }
virtual const char* Name() const override { return kClassName(); }
virtual const char* NickName() const override { return kNickName(); }
private:
HashLinkListRepOptions options_;
};
} // namespace
MemTableRep* HashLinkListRepFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* logger) {
return new HashLinkListRep(compare, allocator, transform, bucket_count_,
threshold_use_skiplist_, huge_page_tlb_size_,
logger, bucket_entries_logging_threshold_,
if_log_bucket_dist_when_flash_);
return new HashLinkListRep(
compare, allocator, transform, options_.bucket_count,
options_.threshold_use_skiplist, options_.huge_page_tlb_size, logger,
options_.bucket_entries_logging_threshold,
options_.if_log_bucket_dist_when_flash);
}
MemTableRepFactory* NewHashLinkListRepFactory(

49
memtable/hash_linklist_rep.h
View File

@ -1,49 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#ifndef ROCKSDB_LITE
#include "rocksdb/slice_transform.h"
#include "rocksdb/memtablerep.h"
namespace ROCKSDB_NAMESPACE {
class HashLinkListRepFactory : public MemTableRepFactory {
public:
explicit HashLinkListRepFactory(size_t bucket_count,
uint32_t threshold_use_skiplist,
size_t huge_page_tlb_size,
int bucket_entries_logging_threshold,
bool if_log_bucket_dist_when_flash)
: bucket_count_(bucket_count),
threshold_use_skiplist_(threshold_use_skiplist),
huge_page_tlb_size_(huge_page_tlb_size),
bucket_entries_logging_threshold_(bucket_entries_logging_threshold),
if_log_bucket_dist_when_flash_(if_log_bucket_dist_when_flash) {}
virtual ~HashLinkListRepFactory() {}
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* logger) override;
virtual const char* Name() const override {
return "HashLinkListRepFactory";
}
private:
const size_t bucket_count_;
const uint32_t threshold_use_skiplist_;
const size_t huge_page_tlb_size_;
int bucket_entries_logging_threshold_;
bool if_log_bucket_dist_when_flash_;
};
} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE

56
memtable/hash_skiplist_rep.cc
View File

@ -5,8 +5,6 @@
//
#ifndef ROCKSDB_LITE
#include "memtable/hash_skiplist_rep.h"
#include <atomic>
#include "db/memtable.h"
@ -16,6 +14,7 @@
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
#include "rocksdb/utilities/options_type.h"
#include "util/murmurhash.h"
namespace ROCKSDB_NAMESPACE {
@ -329,13 +328,60 @@ MemTableRep::Iterator* HashSkipListRep::GetDynamicPrefixIterator(Arena* arena) {
}
}
} // anon namespace
struct HashSkipListRepOptions {
static const char* kName() { return "HashSkipListRepFactoryOptions"; }
size_t bucket_count;
int32_t skiplist_height;
int32_t skiplist_branching_factor;
};
static std::unordered_map<std::string, OptionTypeInfo> hash_skiplist_info = {
{"bucket_count",
{offsetof(struct HashSkipListRepOptions, bucket_count), OptionType::kSizeT,
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
{"skiplist_height",
{offsetof(struct HashSkipListRepOptions, skiplist_height),
OptionType::kInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
{"branching_factor",
{offsetof(struct HashSkipListRepOptions, skiplist_branching_factor),
OptionType::kInt32T, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
};
class HashSkipListRepFactory : public MemTableRepFactory {
public:
explicit HashSkipListRepFactory(size_t bucket_count, int32_t skiplist_height,
int32_t skiplist_branching_factor) {
options_.bucket_count = bucket_count;
options_.skiplist_height = skiplist_height;
options_.skiplist_branching_factor = skiplist_branching_factor;
RegisterOptions(&options_, &hash_skiplist_info);
}
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* logger) override;
static const char* kClassName() { return "HashSkipListRepFactory"; }
static const char* kNickName() { return "prefix_hash"; }
virtual const char* Name() const override { return kClassName(); }
virtual const char* NickName() const override { return kNickName(); }
private:
HashSkipListRepOptions options_;
};
} // namespace
MemTableRep* HashSkipListRepFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* /*logger*/) {
return new HashSkipListRep(compare, allocator, transform, bucket_count_,
skiplist_height_, skiplist_branching_factor_);
return new HashSkipListRep(compare, allocator, transform,
options_.bucket_count, options_.skiplist_height,
options_.skiplist_branching_factor);
}
MemTableRepFactory* NewHashSkipListRepFactory(

44
memtable/hash_skiplist_rep.h
View File

@ -1,44 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#ifndef ROCKSDB_LITE
#include "rocksdb/slice_transform.h"
#include "rocksdb/memtablerep.h"
namespace ROCKSDB_NAMESPACE {
class HashSkipListRepFactory : public MemTableRepFactory {
public:
explicit HashSkipListRepFactory(
size_t bucket_count,
int32_t skiplist_height,
int32_t skiplist_branching_factor)
: bucket_count_(bucket_count),
skiplist_height_(skiplist_height),
skiplist_branching_factor_(skiplist_branching_factor) { }
virtual ~HashSkipListRepFactory() {}
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* logger) override;
virtual const char* Name() const override {
return "HashSkipListRepFactory";
}
private:
const size_t bucket_count_;
const int32_t skiplist_height_;
const int32_t skiplist_branching_factor_;
};
} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE

19
memtable/memtablerep_bench.cc
View File

@ -28,6 +28,7 @@ int main() {
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/comparator.h"
#include "rocksdb/convenience.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/options.h"
#include "rocksdb/slice_transform.h"
@ -581,13 +582,15 @@ int main(int argc, char** argv) {
#ifndef ROCKSDB_LITE
} else if (FLAGS_memtablerep == "vector") {
factory.reset(new ROCKSDB_NAMESPACE::VectorRepFactory);
} else if (FLAGS_memtablerep == "hashskiplist") {
} else if (FLAGS_memtablerep == "hashskiplist" ||
FLAGS_memtablerep == "prefix_hash") {
factory.reset(ROCKSDB_NAMESPACE::NewHashSkipListRepFactory(
FLAGS_bucket_count, FLAGS_hashskiplist_height,
FLAGS_hashskiplist_branching_factor));
options.prefix_extractor.reset(
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length));
} else if (FLAGS_memtablerep == "hashlinklist") {
} else if (FLAGS_memtablerep == "hashlinklist" ||
FLAGS_memtablerep == "hash_linkedlist") {
factory.reset(ROCKSDB_NAMESPACE::NewHashLinkListRepFactory(
FLAGS_bucket_count, FLAGS_huge_page_tlb_size,
FLAGS_bucket_entries_logging_threshold,
@ -596,8 +599,16 @@ int main(int argc, char** argv) {
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length));
#endif // ROCKSDB_LITE
} else {
fprintf(stdout, "Unknown memtablerep: %s\n", FLAGS_memtablerep.c_str());
exit(1);
ROCKSDB_NAMESPACE::ConfigOptions config_options;
config_options.ignore_unsupported_options = false;
ROCKSDB_NAMESPACE::Status s =
ROCKSDB_NAMESPACE::MemTableRepFactory::CreateFromString(
config_options, FLAGS_memtablerep, &factory);
if (!s.ok()) {
fprintf(stdout, "Unknown memtablerep: %s\n", s.ToString().c_str());
exit(1);
}
}
ROCKSDB_NAMESPACE::InternalKeyComparator internal_key_comp(

23
memtable/skiplistrep.cc
View File

@ -9,6 +9,8 @@
#include "memory/arena.h"
#include "memtable/inlineskiplist.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/utilities/options_type.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
namespace {
@ -335,6 +337,27 @@ public:
};
}
static std::unordered_map<std::string, OptionTypeInfo> skiplist_factory_info = {
#ifndef ROCKSDB_LITE
{"lookahead",
{0, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kDontSerialize /*Since it is part of the ID*/}},
#endif
};
SkipListFactory::SkipListFactory(size_t lookahead) : lookahead_(lookahead) {
RegisterOptions("SkipListFactoryOptions", &lookahead_,
&skiplist_factory_info);
}
std::string SkipListFactory::GetId() const {
std::string id = Name();
if (lookahead_ > 0) {
id.append(":").append(ROCKSDB_NAMESPACE::ToString(lookahead_));
}
return id;
}
MemTableRep* SkipListFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* /*logger*/) {

20
memtable/vectorrep.cc
View File

@ -4,18 +4,18 @@
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
#include "rocksdb/memtablerep.h"
#include <unordered_set>
#include <set>
#include <memory>
#include <algorithm>
#include <memory>
#include <set>
#include <type_traits>
#include <unordered_set>
#include "db/memtable.h"
#include "memory/arena.h"
#include "memtable/stl_wrappers.h"
#include "port/port.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/utilities/options_type.h"
#include "util/mutexlock.h"
namespace ROCKSDB_NAMESPACE {
@ -292,6 +292,16 @@ MemTableRep::Iterator* VectorRep::GetIterator(Arena* arena) {
}
} // anon namespace
static std::unordered_map<std::string, OptionTypeInfo> vector_rep_table_info = {
{"count",
{0, OptionType::kSizeT, OptionVerificationType::kNormal,
OptionTypeFlags::kNone}},
};
VectorRepFactory::VectorRepFactory(size_t count) : count_(count) {
RegisterOptions("VectorRepFactoryOptions", &count_, &vector_rep_table_info);
}
MemTableRep* VectorRepFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform*, Logger* /*logger*/) {

34
options/cf_options.cc
View File

@ -573,21 +573,33 @@ static std::unordered_map<std::string, OptionTypeInfo>
OptionTypeFlags::kNone}},
{"memtable_factory",
{offset_of(&ImmutableCFOptions::memtable_factory),
OptionType::kMemTableRepFactory, OptionVerificationType::kByName,
OptionTypeFlags::kNone}},
OptionType::kCustomizable, OptionVerificationType::kByName,
OptionTypeFlags::kShared,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, void* addr) {
std::unique_ptr<MemTableRepFactory> factory;
auto* shared =
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
Status s =
MemTableRepFactory::CreateFromString(opts, value, &factory);
if (s.ok()) {
shared->reset(factory.release());
}
return s;
}}},
{"memtable",
{offset_of(&ImmutableCFOptions::memtable_factory),
OptionType::kMemTableRepFactory, OptionVerificationType::kAlias,
OptionTypeFlags::kNone,
// Parses the value string and updates the memtable_factory
[](const ConfigOptions& /*opts*/, const std::string& /*name*/,
OptionType::kCustomizable, OptionVerificationType::kAlias,
OptionTypeFlags::kShared,
[](const ConfigOptions& opts, const std::string&,
const std::string& value, void* addr) {
std::unique_ptr<MemTableRepFactory> new_mem_factory;
Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory);
std::unique_ptr<MemTableRepFactory> factory;
auto* shared =
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
Status s =
MemTableRepFactory::CreateFromString(opts, value, &factory);
if (s.ok()) {
auto memtable_factory =
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
memtable_factory->reset(new_mem_factory.release());
shared->reset(factory.release());
}
return s;
}}},

17
options/customizable_test.cc
View File

@ -1302,6 +1302,23 @@ TEST_F(LoadCustomizableTest, LoadComparatorTest) {
}
}
TEST_F(LoadCustomizableTest, LoadMemTableRepFactoryTest) {
std::unique_ptr<MemTableRepFactory> result;
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options_, "SpecialSkipListFactory", &result));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options_, SkipListFactory::kClassName(), &result));
ASSERT_NE(result.get(), nullptr);
ASSERT_TRUE(result->IsInstanceOf(SkipListFactory::kClassName()));
if (RegisterTests("Test")) {
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options_, "SpecialSkipListFactory", &result));
ASSERT_NE(result, nullptr);
ASSERT_STREQ(result->Name(), "SpecialSkipListFactory");
}
}
TEST_F(LoadCustomizableTest, LoadMergeOperatorTest) {
std::shared_ptr<MergeOperator> result;

6
options/options_helper.cc
View File

@ -562,12 +562,6 @@ bool SerializeSingleOptionHelper(const void* opt_address,
: kNullptrString;
break;
}
case OptionType::kMemTableRepFactory: {
const auto* ptr =
static_cast<const std::shared_ptr<MemTableRepFactory>*>(opt_address);
*value = ptr->get() ? ptr->get()->Name() : kNullptrString;
break;
}
case OptionType::kFilterPolicy: {
const auto* ptr =
static_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);

108
options/options_test.cc
View File

@ -575,6 +575,7 @@ TEST_F(OptionsTest, GetColumnFamilyOptionsFromStringTest) {
&new_cf_opt));
ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
}
TEST_F(OptionsTest, CompressionOptionsFromString) {
@ -1137,14 +1138,14 @@ TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory));
ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory));
ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory");
ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory");
ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt",
&new_mem_factory));
ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory));
ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000",
&new_mem_factory));
ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory");
ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory");
ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt",
&new_mem_factory));
@ -1170,6 +1171,99 @@ TEST_F(OptionsTest, GetMemTableRepFactoryFromString) {
}
#endif // !ROCKSDB_LITE
TEST_F(OptionsTest, MemTableRepFactoryCreateFromString) {
std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
ConfigOptions config_options;
config_options.ignore_unsupported_options = false;
config_options.ignore_unknown_options = false;
ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list",
&new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list:16",
&new_mem_factory));
ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory");
ASSERT_TRUE(new_mem_factory->IsInstanceOf("skip_list"));
ASSERT_TRUE(new_mem_factory->IsInstanceOf("SkipListFactory"));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "skip_list:16:invalid_opt", &new_mem_factory));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "invalid_opt=10", &new_mem_factory));
// Test a reset
ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "",
&new_mem_factory));
ASSERT_EQ(new_mem_factory, nullptr);
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "invalid_opt=10", &new_mem_factory));
#ifndef ROCKSDB_LITE
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options, "id=skip_list; lookahead=32", &new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "prefix_hash",
&new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options, "prefix_hash:1000", &new_mem_factory));
ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory");
ASSERT_TRUE(new_mem_factory->IsInstanceOf("prefix_hash"));
ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashSkipListRepFactory"));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "prefix_hash:1000:invalid_opt", &new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options,
"id=prefix_hash; bucket_count=32; skiplist_height=64; "
"branching_factor=16",
&new_mem_factory));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options,
"id=prefix_hash; bucket_count=32; skiplist_height=64; "
"branching_factor=16; invalid=unknown",
&new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options, "hash_linkedlist", &new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options, "hash_linkedlist:1000", &new_mem_factory));
ASSERT_STREQ(new_mem_factory->Name(), "HashLinkListRepFactory");
ASSERT_TRUE(new_mem_factory->IsInstanceOf("hash_linkedlist"));
ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashLinkListRepFactory"));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "hash_linkedlist:1000:invalid_opt", &new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options,
"id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; "
"logging_threshold=12; log_when_flash=true",
&new_mem_factory));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options,
"id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; "
"logging_threshold=12; log_when_flash=true; invalid=unknown",
&new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector",
&new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector:1024",
&new_mem_factory));
ASSERT_STREQ(new_mem_factory->Name(), "VectorRepFactory");
ASSERT_TRUE(new_mem_factory->IsInstanceOf("vector"));
ASSERT_TRUE(new_mem_factory->IsInstanceOf("VectorRepFactory"));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "vector:1024:invalid_opt", &new_mem_factory));
ASSERT_OK(MemTableRepFactory::CreateFromString(
config_options, "id=vector; count=42", &new_mem_factory));
ASSERT_NOK(MemTableRepFactory::CreateFromString(
config_options, "id=vector; invalid=unknown", &new_mem_factory));
#endif // ROCKSDB_LITE
ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo",
&new_mem_factory));
// CuckooHash memtable is already removed.
ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo:1024",
&new_mem_factory));
ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "bad_factory",
&new_mem_factory));
}
#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
TEST_F(OptionsTest, GetOptionsFromStringTest) {
Options base_options, new_options;
@ -2454,7 +2548,7 @@ TEST_F(OptionsOldApiTest, GetColumnFamilyOptionsFromStringTest) {
"memtable=skip_list:10;arena_block_size=1024",
&new_cf_opt));
ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
}
TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) {
@ -2650,6 +2744,14 @@ TEST_F(OptionsOldApiTest, GetPlainTableOptionsFromString) {
ASSERT_TRUE(new_opt.full_scan_mode);
ASSERT_TRUE(new_opt.store_index_in_file);
std::unordered_map<std::string, std::string> opt_map;
ASSERT_OK(StringToMap(
"user_key_len=55;bloom_bits_per_key=10;huge_page_tlb_size=8;", &opt_map));
ASSERT_OK(GetPlainTableOptionsFromMap(table_opt, opt_map, &new_opt));
ASSERT_EQ(new_opt.user_key_len, 55u);
ASSERT_EQ(new_opt.bloom_bits_per_key, 10);
ASSERT_EQ(new_opt.huge_page_tlb_size, 8);
// unknown option
ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
"user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"

206
table/plain/plain_table_factory.cc
View File

@ -3,7 +3,6 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "table/plain/plain_table_factory.h"
#include <stdint.h>
@ -11,15 +10,17 @@
#include <memory>
#include "db/dbformat.h"
#include "options/configurable_helper.h"
#include "port/port.h"
#include "rocksdb/convenience.h"
#include "rocksdb/utilities/customizable_util.h"
#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "table/plain/plain_table_builder.h"
#include "table/plain/plain_table_reader.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
#ifndef ROCKSDB_LITE
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
@ -151,73 +152,146 @@ Status GetPlainTableOptionsFromString(const ConfigOptions& config_options,
return Status::InvalidArgument(s.getState());
}
}
#endif // ROCKSDB_LITE
#ifndef ROCKSDB_LITE
static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library,
const std::string& /*arg*/) {
// The MemTableRepFactory built-in classes will be either a class
// (VectorRepFactory) or a nickname (vector), followed optionally by ":#",
// where # is the "size" of the factory.
auto AsRegex = [](const std::string& name, const std::string& alt) {
std::string regex;
regex.append("(").append(name);
regex.append("|").append(alt).append(")(:[0-9]*)?");
return regex;
};
library.Register<MemTableRepFactory>(
AsRegex(VectorRepFactory::kClassName(), VectorRepFactory::kNickName()),
[](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
std::string* /*errmsg*/) {
auto colon = uri.find(":");
if (colon != std::string::npos) {
size_t count = ParseSizeT(uri.substr(colon + 1));
guard->reset(new VectorRepFactory(count));
} else {
guard->reset(new VectorRepFactory());
}
return guard->get();
});
library.Register<MemTableRepFactory>(
AsRegex(SkipListFactory::kClassName(), SkipListFactory::kNickName()),
[](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
std::string* /*errmsg*/) {
auto colon = uri.find(":");
if (colon != std::string::npos) {
size_t lookahead = ParseSizeT(uri.substr(colon + 1));
guard->reset(new SkipListFactory(lookahead));
} else {
guard->reset(new SkipListFactory());
}
return guard->get();
});
library.Register<MemTableRepFactory>(
AsRegex("HashLinkListRepFactory", "hash_linkedlist"),
[](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
std::string* /*errmsg*/) {
// Expecting format: hash_linkedlist:<hash_bucket_count>
auto colon = uri.find(":");
if (colon != std::string::npos) {
size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1));
guard->reset(NewHashLinkListRepFactory(hash_bucket_count));
} else {
guard->reset(NewHashLinkListRepFactory());
}
return guard->get();
});
library.Register<MemTableRepFactory>(
AsRegex("HashSkipListRepFactory", "prefix_hash"),
[](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
std::string* /*errmsg*/) {
// Expecting format: prefix_hash:<hash_bucket_count>
auto colon = uri.find(":");
if (colon != std::string::npos) {
size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1));
guard->reset(NewHashSkipListRepFactory(hash_bucket_count));
} else {
guard->reset(NewHashSkipListRepFactory());
}
return guard->get();
});
library.Register<MemTableRepFactory>(
"cuckoo",
[](const std::string& /*uri*/,
std::unique_ptr<MemTableRepFactory>* /*guard*/, std::string* errmsg) {
*errmsg = "cuckoo hash memtable is not supported anymore.";
return nullptr;
});
return 5;
}
#endif // ROCKSDB_LITE
Status GetMemTableRepFactoryFromString(
const std::string& opts_str,
std::unique_ptr<MemTableRepFactory>* new_mem_factory) {
std::vector<std::string> opts_list = StringSplit(opts_str, ':');
size_t len = opts_list.size();
if (opts_list.empty() || opts_list.size() > 2) {
return Status::InvalidArgument("Can't parse memtable_factory option ",
opts_str);
}
MemTableRepFactory* mem_factory = nullptr;
if (opts_list[0] == "skip_list" || opts_list[0] == "SkipListFactory") {
// Expecting format
// skip_list:<lookahead>
if (2 == len) {
size_t lookahead = ParseSizeT(opts_list[1]);
mem_factory = new SkipListFactory(lookahead);
} else if (1 == len) {
mem_factory = new SkipListFactory();
}
} else if (opts_list[0] == "prefix_hash" ||
opts_list[0] == "HashSkipListRepFactory") {
// Expecting format
// prfix_hash:<hash_bucket_count>
if (2 == len) {
size_t hash_bucket_count = ParseSizeT(opts_list[1]);
mem_factory = NewHashSkipListRepFactory(hash_bucket_count);
} else if (1 == len) {
mem_factory = NewHashSkipListRepFactory();
}
} else if (opts_list[0] == "hash_linkedlist" ||
opts_list[0] == "HashLinkListRepFactory") {
// Expecting format
// hash_linkedlist:<hash_bucket_count>
if (2 == len) {
size_t hash_bucket_count = ParseSizeT(opts_list[1]);
mem_factory = NewHashLinkListRepFactory(hash_bucket_count);
} else if (1 == len) {
mem_factory = NewHashLinkListRepFactory();
}
} else if (opts_list[0] == "vector" || opts_list[0] == "VectorRepFactory") {
// Expecting format
// vector:<count>
if (2 == len) {
size_t count = ParseSizeT(opts_list[1]);
mem_factory = new VectorRepFactory(count);
} else if (1 == len) {
mem_factory = new VectorRepFactory();
}
} else if (opts_list[0] == "cuckoo") {
return Status::NotSupported(
"cuckoo hash memtable is not supported anymore.");
} else {
return Status::InvalidArgument("Unrecognized memtable_factory option ",
opts_str);
}
if (mem_factory != nullptr) {
new_mem_factory->reset(mem_factory);
}
return Status::OK();
const std::string& opts_str, std::unique_ptr<MemTableRepFactory>* result) {
ConfigOptions config_options;
config_options.ignore_unsupported_options = false;
config_options.ignore_unknown_options = false;
return MemTableRepFactory::CreateFromString(config_options, opts_str, result);
}
Status MemTableRepFactory::CreateFromString(
const ConfigOptions& config_options, const std::string& value,
std::unique_ptr<MemTableRepFactory>* result) {
#ifndef ROCKSDB_LITE
static std::once_flag once;
std::call_once(once, [&]() {
RegisterBuiltinMemTableRepFactory(*(ObjectLibrary::Default().get()), "");
});
#endif // ROCKSDB_LITE
std::string id;
std::unordered_map<std::string, std::string> opt_map;
Status status = Customizable::GetOptionsMap(config_options, result->get(),
value, &id, &opt_map);
if (!status.ok()) { // GetOptionsMap failed
return status;
} else if (value.empty()) {
// No Id and no options. Clear the object
result->reset();
return Status::OK();
} else if (id.empty()) { // We have no Id but have options. Not good
return Status::NotSupported("Cannot reset object ", id);
} else {
#ifndef ROCKSDB_LITE
status = NewUniqueObject<MemTableRepFactory>(config_options, id, opt_map,
result);
#else
// To make it possible to configure the memtables in LITE mode, the ID
// is of the form <name>:<size>, where name is the name of the class and
// <size> is the length of the object (e.g. skip_list:10).
std::vector<std::string> opts_list = StringSplit(id, ':');
if (opts_list.empty() || opts_list.size() > 2 || !opt_map.empty()) {
status = Status::InvalidArgument("Can't parse memtable_factory option ",
value);
} else if (opts_list[0] == "skip_list" ||
opts_list[0] == SkipListFactory::kClassName()) {
// Expecting format
// skip_list:<lookahead>
if (opts_list.size() == 2) {
size_t lookahead = ParseSizeT(opts_list[1]);
result->reset(new SkipListFactory(lookahead));
} else {
result->reset(new SkipListFactory());
}
} else if (!config_options.ignore_unsupported_options) {
status = Status::NotSupported("Cannot load object in LITE mode ", id);
}
#endif // ROCKSDB_LITE
}
return status;
}
#ifndef ROCKSDB_LITE
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
@ -259,5 +333,5 @@ const std::string PlainTablePropertyNames::kBloomVersion =
const std::string PlainTablePropertyNames::kNumBloomBlocks =
"rocksdb.plain.table.bloom.numblocks";
} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE

126
test_util/testutil.cc
View File

@ -601,6 +601,106 @@ Status CreateEnvFromSystem(const ConfigOptions& config_options, Env** result,
return Status::OK();
}
}
namespace {
// A hacky skip list mem table that triggers flush after number of entries.
class SpecialMemTableRep : public MemTableRep {
public:
explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
int num_entries_flush)
: MemTableRep(allocator),
memtable_(memtable),
num_entries_flush_(num_entries_flush),
num_entries_(0) {}
virtual KeyHandle Allocate(const size_t len, char** buf) override {
return memtable_->Allocate(len, buf);
}
// Insert key into the list.
// REQUIRES: nothing that compares equal to key is currently in the list.
virtual void Insert(KeyHandle handle) override {
num_entries_++;
memtable_->Insert(handle);
}
void InsertConcurrently(KeyHandle handle) override {
num_entries_++;
memtable_->Insert(handle);
}
// Returns true iff an entry that compares equal to key is in the list.
virtual bool Contains(const char* key) const override {
return memtable_->Contains(key);
}
virtual size_t ApproximateMemoryUsage() override {
// Return a high memory usage when number of entries exceeds the threshold
// to trigger a flush.
return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024;
}
virtual void Get(const LookupKey& k, void* callback_args,
bool (*callback_func)(void* arg,
const char* entry)) override {
memtable_->Get(k, callback_args, callback_func);
}
uint64_t ApproximateNumEntries(const Slice& start_ikey,
const Slice& end_ikey) override {
return memtable_->ApproximateNumEntries(start_ikey, end_ikey);
}
virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override {
return memtable_->GetIterator(arena);
}
virtual ~SpecialMemTableRep() override {}
private:
std::unique_ptr<MemTableRep> memtable_;
int num_entries_flush_;
int num_entries_;
};
class SpecialSkipListFactory : public MemTableRepFactory {
public:
// After number of inserts exceeds `num_entries_flush` in a mem table, trigger
// flush.
explicit SpecialSkipListFactory(int num_entries_flush)
: num_entries_flush_(num_entries_flush) {}
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* /*logger*/) override {
return new SpecialMemTableRep(
allocator,
factory_.CreateMemTableRep(compare, allocator, transform, nullptr),
num_entries_flush_);
}
static const char* kClassName() { return "SpecialSkipListFactory"; }
virtual const char* Name() const override { return kClassName(); }
std::string GetId() const override {
std::string id = Name();
if (num_entries_flush_ > 0) {
id.append(":").append(ROCKSDB_NAMESPACE::ToString(num_entries_flush_));
}
return id;
}
bool IsInsertConcurrentlySupported() const override {
return factory_.IsInsertConcurrentlySupported();
}
private:
SkipListFactory factory_;
int num_entries_flush_;
};
} // namespace
MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush) {
RegisterTestLibrary();
return new SpecialSkipListFactory(num_entries_per_flush);
}
#ifndef ROCKSDB_LITE
// This method loads existing test classes into the ObjectRegistry
@ -614,6 +714,19 @@ int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/) {
static test::SimpleSuffixReverseComparator ssrc;
return &ssrc;
});
library.Register<MemTableRepFactory>(
std::string(SpecialSkipListFactory::kClassName()) + "(:[0-9]*)?",
[](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
std::string* /* errmsg */) {
auto colon = uri.find(":");
if (colon != std::string::npos) {
auto count = ParseInt(uri.substr(colon + 1));
guard->reset(new SpecialSkipListFactory(count));
} else {
guard->reset(new SpecialSkipListFactory(2));
}
return guard->get();
});
library.Register<MergeOperator>(
"Changling",
[](const std::string& uri, std::unique_ptr<MergeOperator>* guard,
@ -637,6 +750,19 @@ int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/) {
return static_cast<int>(library.GetFactoryCount(&num_types));
}
#endif // ROCKSDB_LITE
void RegisterTestLibrary(const std::string& arg) {
static bool registered = false;
if (!registered) {
registered = true;
#ifndef ROCKSDB_LITE
ObjectRegistry::Default()->AddLibrary("test", RegisterTestObjects, arg);
#else
(void)arg;
#endif // ROCKSDB_LITE
}
}
} // namespace test
} // namespace ROCKSDB_NAMESPACE

8
test_util/testutil.h
View File

@ -27,6 +27,7 @@
namespace ROCKSDB_NAMESPACE {
class FileSystem;
class MemTableRepFactory;
class ObjectLibrary;
class Random;
class SequentialFile;
@ -853,6 +854,10 @@ class ChanglingCompactionFilterFactory : public CompactionFilterFactory {
std::string name_;
};
// The factory for the hacky skip list mem table that triggers flush after
// number of entries exceeds a threshold.
extern MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush);
extern const Comparator* ComparatorWithU64Ts();
CompressionType RandomCompressionType(Random* rnd);
@ -901,5 +906,8 @@ Status CreateEnvFromSystem(const ConfigOptions& options, Env** result,
// Registers the testutil classes with the ObjectLibrary
int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/);
#endif // ROCKSDB_LITE
// Register the testutil classes with the default ObjectRegistry/Library
void RegisterTestLibrary(const std::string& arg = "");
} // namespace test
} // namespace ROCKSDB_NAMESPACE

122
tools/db_bench_tool.cc
View File

@ -1447,30 +1447,6 @@ DEFINE_int64(multiread_stride, 0,
"Stride length for the keys in a MultiGet batch");
DEFINE_bool(multiread_batched, false, "Use the new MultiGet API");
enum RepFactory {
kSkipList,
kPrefixHash,
kVectorRep,
kHashLinkedList,
};
static enum RepFactory StringToRepFactory(const char* ctype) {
assert(ctype);
if (!strcasecmp(ctype, "skip_list"))
return kSkipList;
else if (!strcasecmp(ctype, "prefix_hash"))
return kPrefixHash;
else if (!strcasecmp(ctype, "vector"))
return kVectorRep;
else if (!strcasecmp(ctype, "hash_linkedlist"))
return kHashLinkedList;
fprintf(stdout, "Cannot parse memreptable %s\n", ctype);
return kSkipList;
}
static enum RepFactory FLAGS_rep_factory;
DEFINE_string(memtablerep, "skip_list", "");
DEFINE_int64(hash_bucket_count, 1024 * 1024, "hash bucket count");
DEFINE_bool(use_plain_table, false, "if use plain table "
@ -1534,8 +1510,33 @@ static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((__unused__
&ValidateTableCacheNumshardbits);
namespace ROCKSDB_NAMESPACE {
namespace {
static Status CreateMemTableRepFactory(
const ConfigOptions& config_options,
std::shared_ptr<MemTableRepFactory>* factory) {
Status s;
if (!strcasecmp(FLAGS_memtablerep.c_str(), SkipListFactory::kNickName())) {
factory->reset(new SkipListFactory(FLAGS_skip_list_lookahead));
#ifndef ROCKSDB_LITE
} else if (!strcasecmp(FLAGS_memtablerep.c_str(), "prefix_hash")) {
factory->reset(NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
} else if (!strcasecmp(FLAGS_memtablerep.c_str(),
VectorRepFactory::kNickName())) {
factory->reset(new VectorRepFactory());
} else if (!strcasecmp(FLAGS_memtablerep.c_str(), "hash_linkedlist")) {
factory->reset(NewHashLinkListRepFactory(FLAGS_hash_bucket_count));
#endif // ROCKSDB_LITE
} else {
std::unique_ptr<MemTableRepFactory> unique;
s = MemTableRepFactory::CreateFromString(config_options, FLAGS_memtablerep,
&unique);
if (s.ok()) {
factory->reset(unique.release());
}
}
return s;
}
struct ReportFileOpCounters {
std::atomic<int> open_counter_;
std::atomic<int> delete_counter_;
@ -2681,7 +2682,7 @@ class Benchmark {
compressed);
}
void PrintHeader() {
void PrintHeader(const Options& options) {
PrintEnvironment();
fprintf(stdout,
"Keys: %d bytes each (+ %d bytes user-defined timestamp)\n",
@ -2731,20 +2732,9 @@ class Benchmark {
fprintf(stdout, "Compression: %s\n", compression.c_str());
fprintf(stdout, "Compression sampling rate: %" PRId64 "\n",
FLAGS_sample_for_compression);
switch (FLAGS_rep_factory) {
case kPrefixHash:
fprintf(stdout, "Memtablerep: prefix_hash\n");
break;
case kSkipList:
fprintf(stdout, "Memtablerep: skip_list\n");
break;
case kVectorRep:
fprintf(stdout, "Memtablerep: vector\n");
break;
case kHashLinkedList:
fprintf(stdout, "Memtablerep: hash_linkedlist\n");
break;
if (options.memtable_factory != nullptr) {
fprintf(stdout, "Memtablerep: %s\n",
options.memtable_factory->GetId().c_str());
}
fprintf(stdout, "Perf Level: %d\n", FLAGS_perf_level);
@ -3209,7 +3199,7 @@ class Benchmark {
ErrorExit();
}
Open(&open_options_);
PrintHeader();
PrintHeader(open_options_);
std::stringstream benchmark_stream(FLAGS_benchmarks);
std::string name;
std::unique_ptr<ExpiredTimeFilter> filter;
@ -3932,6 +3922,7 @@ class Benchmark {
printf("Initializing RocksDB Options from command-line flags\n");
Options& options = *opts;
ConfigOptions config_options(options);
config_options.ignore_unsupported_options = false;
assert(db_.db == nullptr);
@ -4007,42 +3998,25 @@ class Benchmark {
FLAGS_level_compaction_dynamic_level_bytes;
options.max_bytes_for_level_multiplier =
FLAGS_max_bytes_for_level_multiplier;
if ((FLAGS_prefix_size == 0) && (FLAGS_rep_factory == kPrefixHash ||
FLAGS_rep_factory == kHashLinkedList)) {
Status s =
CreateMemTableRepFactory(config_options, &options.memtable_factory);
if (!s.ok()) {
fprintf(stderr, "Could not create memtable factory: %s\n",
s.ToString().c_str());
exit(1);
} else if ((FLAGS_prefix_size == 0) &&
(options.memtable_factory->IsInstanceOf("prefix_hash") ||
options.memtable_factory->IsInstanceOf("hash_linkedlist"))) {
fprintf(stderr, "prefix_size should be non-zero if PrefixHash or "
"HashLinkedList memtablerep is used\n");
exit(1);
}
switch (FLAGS_rep_factory) {
case kSkipList:
options.memtable_factory.reset(new SkipListFactory(
FLAGS_skip_list_lookahead));
break;
#ifndef ROCKSDB_LITE
case kPrefixHash:
options.memtable_factory.reset(
NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
break;
case kHashLinkedList:
options.memtable_factory.reset(NewHashLinkListRepFactory(
FLAGS_hash_bucket_count));
break;
case kVectorRep:
options.memtable_factory.reset(
new VectorRepFactory
);
break;
#else
default:
fprintf(stderr, "Only skip list is supported in lite mode\n");
exit(1);
#endif // ROCKSDB_LITE
}
if (FLAGS_use_plain_table) {
#ifndef ROCKSDB_LITE
if (FLAGS_rep_factory != kPrefixHash &&
FLAGS_rep_factory != kHashLinkedList) {
fprintf(stderr, "Waring: plain table is used with skipList\n");
if (!options.memtable_factory->IsInstanceOf("prefix_hash") &&
!options.memtable_factory->IsInstanceOf("hash_linkedlist")) {
fprintf(stderr, "Warning: plain table is used with %s\n",
options.memtable_factory->Name());
}
int bloom_bits_per_key = FLAGS_bloom_bits;
@ -4298,8 +4272,8 @@ class Benchmark {
// merge operator options
if (!FLAGS_merge_operator.empty()) {
Status s = MergeOperator::CreateFromString(
config_options, FLAGS_merge_operator, &options.merge_operator);
s = MergeOperator::CreateFromString(config_options, FLAGS_merge_operator,
&options.merge_operator);
if (!s.ok()) {
fprintf(stderr, "invalid merge operator[%s]: %s\n",
FLAGS_merge_operator.c_str(), s.ToString().c_str());
@ -8148,8 +8122,6 @@ int db_bench_tool(int argc, char** argv) {
FLAGS_value_size_distribution_type_e =
StringToDistributionType(FLAGS_value_size_distribution_type.c_str());
FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
// Note options sanitization may increase thread pool sizes according to
// max_background_flushes/max_background_compactions/max_background_jobs
FLAGS_env->SetBackgroundThreads(FLAGS_num_high_pri_threads,