From 0a5afd1afc1428f960190e0a6beb68ee5ed44f49 Mon Sep 17 00:00:00 2001 From: Xing Jin Date: Tue, 13 Aug 2013 13:58:02 -0700 Subject: [PATCH] Minor fix to current codes Summary: Minor fix to current codes, including: coding style, output format, comments. No major logic change. There are only 2 real changes, please see my inline comments. Test Plan: make all check Reviewers: haobo, dhruba, emayanke Differential Revision: https://reviews.facebook.net/D12297 --- db/db_bench.cc | 93 +++++++++++++++++++++++++--------------------- db/db_impl.cc | 8 ++-- tools/db_stress.cc | 59 ++++++++++++++--------------- 3 files changed, 84 insertions(+), 76 deletions(-) diff --git a/db/db_bench.cc b/db/db_bench.cc index d7d9a9547..455a1469a 100644 --- a/db/db_bench.cc +++ b/db/db_bench.cc @@ -126,9 +126,9 @@ static int FLAGS_max_write_buffer_number = 0; // The minimum number of write buffers that will be merged together // before writing to storage. This is cheap because it is an // in-memory merge. If this feature is not enabled, then all these -// write buffers are fushed to L0 as separate files and this increases +// write buffers are flushed to L0 as separate files and this increases // read amplification because a get request has to check in all of these -// files. Also, an in-memory merge may result in writing lesser +// files. Also, an in-memory merge may result in writing less // data to storage if there are duplicate records in each of these // individual write buffers. static int FLAGS_min_write_buffer_number_to_merge = 0; @@ -141,10 +141,12 @@ static int FLAGS_max_background_compactions = 0; // style of compaction: level-based vs universal static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel; -// Percentage flexibilty while comparing file size. +// Percentage flexibility while comparing file size +// (for universal compaction only). static int FLAGS_universal_size_ratio = 1; -// The minimum number of files in a single compaction run. +// The minimum number of files in a single compaction run +// (for universal compaction only). static int FLAGS_compaction_universal_min_merge_width = 2; // Number of bytes to use as a cache of uncompressed data. @@ -212,16 +214,16 @@ static bool FLAGS_get_approx = false; // The total number of levels static int FLAGS_num_levels = 7; -// Target level-0 file size for compaction +// Target file size at level-1 static int FLAGS_target_file_size_base = 2 * 1048576; -// A multiplier to compute targe level-N file size +// A multiplier to compute target level-N file size (N >= 2) static int FLAGS_target_file_size_multiplier = 1; // Max bytes for level-1 static uint64_t FLAGS_max_bytes_for_level_base = 10 * 1048576; -// A multiplier to compute max bytes for level-N +// A multiplier to compute max bytes for level-N (N >= 2) static int FLAGS_max_bytes_for_level_multiplier = 10; // A vector that specifies additional fanout per level @@ -236,16 +238,19 @@ static int FLAGS_level0_slowdown_writes_trigger = 8; // Number of files in level-0 when compactions start static int FLAGS_level0_file_num_compaction_trigger = 4; -// Ratio of reads to writes (expressed as a percentage) -// for the ReadRandomWriteRandom workload. The default -// setting is 9 gets for every 1 put. +// Ratio of reads to reads/writes (expressed as percentage) for the +// ReadRandomWriteRandom workload. The default value 90 means 90% operations +// out of all reads and writes operations are reads. In other words, 9 gets +// for every 1 put. static int FLAGS_readwritepercent = 90; -// This percent of deletes are done (used in RandomWithVerify only) -// Must be smaller than total writepercent (i.e 100 - FLAGS_readwritepercent) +// Percentage of deletes out of reads/writes/deletes (used in RandomWithVerify +// only). RandomWithVerify calculates writepercent as +// (100 - FLAGS_readwritepercent - FLAGS_deletepercent), so FLAGS_deletepercent +// must be smaller than (100 - FLAGS_readwritepercent) static int FLAGS_deletepercent = 2; -// Option to disable compation triggered by read. +// Option to disable compaction triggered by read. static int FLAGS_disable_seek_compaction = false; // Option to delete obsolete files periodically @@ -253,12 +258,13 @@ static int FLAGS_disable_seek_compaction = false; // deleted after every compaction run. static uint64_t FLAGS_delete_obsolete_files_period_micros = 0; -// Algorithm to use to compress the database +// Algorithm used to compress the database static enum leveldb::CompressionType FLAGS_compression_type = leveldb::kSnappyCompression; -// Allows compression for levels 0 and 1 to be disabled when -// other levels are compressed +// If non-negative, compression starts from this level. Levels with number +// < FLAGS_min_level_to_compress are not compressed. +// Otherwise, apply FLAGS_compression_type to all levels. static int FLAGS_min_level_to_compress = -1; static int FLAGS_table_cache_numshardbits = 4; @@ -295,8 +301,8 @@ static bool FLAGS_read_only = false; // Do not auto trigger compactions static bool FLAGS_disable_auto_compactions = false; -// Cap the size of data in levelK for a compaction run -// that compacts Levelk with LevelK+1 +// Cap the size of data in level-K for a compaction run +// that compacts Level-K with Level-(K+1) (for K >= 1) static int FLAGS_source_compaction_factor = 1; // Set the TTL for the WAL Files. @@ -376,6 +382,7 @@ class RandomGenerator { return Slice(data_.data() + pos_ - len, len); } }; + static Slice TrimSpace(Slice s) { unsigned int start = 0; while (start < s.size() && isspace(s[start])) { @@ -784,20 +791,20 @@ class Benchmark { delete db_; delete filter_policy_; } -//this function will construct string format for key. e.g "%016d" -void ConstructStrFormatForKey(char* str, int keySize) -{ - str[0] = '%'; - str[1] = '0'; - sprintf(str+2, "%dd%s", keySize, "%s"); -} -unique_ptr GenerateKeyFromInt(int v, const char* suffix = "") -{ - unique_ptr keyInStr(new char[MAX_KEY_SIZE]); - snprintf(keyInStr.get(), MAX_KEY_SIZE, keyFormat_, v, suffix); - return keyInStr; -} + //this function will construct string format for key. e.g "%016d" + void ConstructStrFormatForKey(char* str, int keySize) { + str[0] = '%'; + str[1] = '0'; + sprintf(str+2, "%dd%s", keySize, "%s"); + } + + unique_ptr GenerateKeyFromInt(int v, const char* suffix = "") { + unique_ptr keyInStr(new char[MAX_KEY_SIZE]); + snprintf(keyInStr.get(), MAX_KEY_SIZE, keyFormat_, v, suffix); + return keyInStr; + } + void Run() { PrintHeader(); Open(); @@ -814,7 +821,7 @@ unique_ptr GenerateKeyFromInt(int v, const char* suffix = "") benchmarks = sep + 1; } - // Reset parameters that may be overriddden bwlow + // Sanitize parameters num_ = FLAGS_num; reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); writes_ = (FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes); @@ -1378,7 +1385,7 @@ unique_ptr GenerateKeyFromInt(int v, const char* suffix = "") // Recalculate number of keys per group, and call MultiGet until done long num_keys; while(num_keys = std::min(keys_left, kpg), !duration.Done(num_keys)) { - found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num,""); + found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num, ""); thread->stats.FinishedSingleOp(db_); keys_left -= num_keys; } @@ -1432,7 +1439,7 @@ unique_ptr GenerateKeyFromInt(int v, const char* suffix = "") thread->stats.AddMessage(msg); } -void ReadMissing(ThreadState* thread) { + void ReadMissing(ThreadState* thread) { FLAGS_warn_missing_keys = false; // Never warn about missing keys Duration duration(FLAGS_duration, reads_); @@ -1446,10 +1453,12 @@ void ReadMissing(ThreadState* thread) { long num_keys; long found; while(num_keys = std::min(keys_left, kpg), !duration.Done(num_keys)) { - found = MultiGetRandom(options, num_keys, thread->rand, FLAGS_num,"."); - if (!found) { - assert(false); - } + found = MultiGetRandom(options, num_keys, thread->rand, FLAGS_num, "."); + + // We should not find any key since the key we try to get has a + // different suffix + assert(!found); + thread->stats.FinishedSingleOp(db_); keys_left -= num_keys; } @@ -1749,10 +1758,8 @@ void ReadMissing(ThreadState* thread) { thread->stats.AddMessage(msg); } - // - // This is diffferent from ReadWhileWriting because it does not use + // This is different from ReadWhileWriting because it does not use // an extra thread. - // void ReadRandomWriteRandom(ThreadState* thread) { if (FLAGS_use_multiget){ // Separate function for multiget (for ease of reading) @@ -1775,7 +1782,7 @@ void ReadMissing(ThreadState* thread) { const int k = thread->rand.Next() % FLAGS_num; unique_ptr key = GenerateKeyFromInt(k); if (get_weight == 0 && put_weight == 0) { - // one batch complated, reinitialize for next batch + // one batch completed, reinitialize for next batch get_weight = FLAGS_readwritepercent; put_weight = 100 - get_weight; } @@ -1876,7 +1883,7 @@ void ReadMissing(ThreadState* thread) { assert(num_keys + num_put_keys <= keys_left); // Apply the MultiGet operations - found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num,""); + found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num, ""); ++multigets_done; reads_done+=num_keys; thread->stats.FinishedSingleOp(db_); diff --git a/db/db_impl.cc b/db/db_impl.cc index a06bc3b9e..e72445183 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -520,11 +520,11 @@ void DBImpl::DeleteObsoleteFiles() { void DBImpl::PurgeObsoleteWALFiles() { int64_t current_time; Status s = env_->GetCurrentTime(¤t_time); - uint64_t now_micros = static_cast(current_time); + uint64_t now_seconds = static_cast(current_time); assert(s.ok()); if (options_.WAL_ttl_seconds != ULONG_MAX && options_.WAL_ttl_seconds > 0) { - if (purge_wal_files_last_run_ + options_.WAL_ttl_seconds > now_micros) { + if (purge_wal_files_last_run_ + options_.WAL_ttl_seconds > now_seconds) { return; } std::vector wal_files; @@ -534,7 +534,7 @@ void DBImpl::PurgeObsoleteWALFiles() { uint64_t file_m_time; const std::string file_path = archival_dir + "/" + f; const Status s = env_->GetFileModificationTime(file_path, &file_m_time); - if (s.ok() && (now_micros - file_m_time > options_.WAL_ttl_seconds)) { + if (s.ok() && (now_seconds - file_m_time > options_.WAL_ttl_seconds)) { Status status = env_->DeleteFile(file_path); if (!status.ok()) { Log(options_.info_log, @@ -544,7 +544,7 @@ void DBImpl::PurgeObsoleteWALFiles() { } // Ignore errors. } } - purge_wal_files_last_run_ = now_micros; + purge_wal_files_last_run_ = now_seconds; } // If externalTable is set, then apply recovered transactions diff --git a/tools/db_stress.cc b/tools/db_stress.cc index 349d22ec5..941b326d3 100644 --- a/tools/db_stress.cc +++ b/tools/db_stress.cc @@ -2,17 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. See the AUTHORS file for names of contributors. // -//The test uses an array to compare against values written to the database. -//Keys written to the array are in 1:1 correspondence to the actual values in -//the database according to the formula in the function GenerateValue +// The test uses an array to compare against values written to the database. +// Keys written to the array are in 1:1 correspondence to the actual values in +// the database according to the formula in the function GenerateValue. -//Space is reserved in the array from 0 to FLAGS_max_key and values are randomly -//written/deleted/read from those positions. During verification we compare all -//the positions in the array. Therefore to shorten/elongate the amount of time -//that this test runs for, you should change the settings: -//FLAGS_max_key, FLAGS_ops_per_thread, (sometimes also FLAGS_threads) -//NOTE that if FLAGS_test_batches_snapshots is set, the test behaves a little -//differently. See comment header for the flag. +// Space is reserved in the array from 0 to FLAGS_max_key and values are +// randomly written/deleted/read from those positions. During verification we +// compare all the positions in the array. To shorten/elongate the running +// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread, +// (sometimes also FLAGS_threads). +// +// NOTE that if FLAGS_test_batches_snapshots is set, the test will have +// different behavior. See comment of the flag for details. #include #include @@ -42,17 +43,17 @@ static const long KB = 1024; static uint32_t FLAGS_seed = 2341234; // Max number of key/values to place in database -static long FLAGS_max_key = 2 * KB * KB * KB; +static long FLAGS_max_key = 1 * KB * KB * KB; -// If set, the test uses MultiGet, MultiPrefixScan, MultiPut and -// MultiDelete that do a different kind of validation during the test -// itself, rather than at the end. This is meant to solve the -// following problems at the expense of doing less degree of -// validation. -// (a) No need to acquire mutexes during writes (less cache flushes in -// multi-core leading to speed up) +// If set, the test uses MultiGet(), MultiPut() and MultiDelete() which +// read/write/delete multiple keys in a batch. In this mode, we do not verify +// db content by comparing the content with the pre-allocated array. Instead, +// we do partial verification inside MultiGet() by checking various values in +// a batch. Benefit of this mode: +// (a) No need to acquire mutexes during writes (less cache flushes +// in multi-core leading to speed up) // (b) No long validation at the end (more speed up) -// (c) Also test snapshot and atomicity of batch writes +// (c) Test snapshot and atomicity of batch writes static bool FLAGS_test_batches_snapshots = false; // Number of concurrent threads to run. @@ -60,7 +61,7 @@ static int FLAGS_threads = 32; // Opens the db with this ttl value if this is not -1 // Carefully specify a large value such that verifications on deleted -// values don't fail +// values don't fail static int FLAGS_ttl = -1; // Size of each value will be this number times rand_int(1,3) bytes @@ -137,16 +138,16 @@ extern int leveldb_kill_odds; // If true, do not write WAL for write. static bool FLAGS_disable_wal = false; -// Target level-0 file size for compaction +// Target level-1 file size for compaction static int FLAGS_target_file_size_base = 64 * KB; -// A multiplier to compute targe level-N file size +// A multiplier to compute targe level-N file size (N >= 2) static int FLAGS_target_file_size_multiplier = 1; -// Max bytes for level-0 +// Max bytes for level-1 static uint64_t FLAGS_max_bytes_for_level_base = 256 * KB; -// A multiplier to compute max bytes for level-N +// A multiplier to compute max bytes for level-N (N >= 2) static int FLAGS_max_bytes_for_level_multiplier = 2; // Number of files in level-0 that will trigger put stop. @@ -341,7 +342,8 @@ class Stats { "", bytes_mb, rate, (100*writes_)/done_, done_); fprintf(stdout, "%-12s: Wrote %ld times\n", "", writes_); fprintf(stdout, "%-12s: Deleted %ld times\n", "", deletes_); - fprintf(stdout, "%-12s: %ld/%ld gets found the key\n", "", founds_, gets_); + fprintf(stdout, "%-12s: %ld read and %ld found the key\n", "", + gets_, founds_); fprintf(stdout, "%-12s: Prefix scanned %ld times\n", "", prefixes_); fprintf(stdout, "%-12s: Iterator size sum is %ld\n", "", iterator_size_sums_); @@ -690,7 +692,6 @@ class StressTest { return s; } - // Given a key K, this deletes ("0"+K), ("1"+K),... ("9"+K) // in DB atomically i.e in a single batch. Also refer MultiGet. Status MultiDelete(ThreadState* thread, @@ -1052,12 +1053,12 @@ class StressTest { fprintf(stdout, "Write-buffer-size : %d\n", FLAGS_write_buffer_size); fprintf(stdout, "Delete percentage : %d\n", FLAGS_delpercent); fprintf(stdout, "Max key : %ld\n", FLAGS_max_key); - fprintf(stdout, "Ratio #ops/#keys : %ld\n", - (FLAGS_ops_per_thread * FLAGS_threads)/FLAGS_max_key); + fprintf(stdout, "Ratio #ops/#keys : %f\n", + (1.0 * FLAGS_ops_per_thread * FLAGS_threads)/FLAGS_max_key); fprintf(stdout, "Num times DB reopens: %d\n", FLAGS_reopen); fprintf(stdout, "Batches/snapshots : %d\n", FLAGS_test_batches_snapshots); - fprintf(stdout, "Purge redundant %% : %d\n", + fprintf(stdout, "Purge redundant %% : %d\n", FLAGS_purge_redundant_percent); fprintf(stdout, "Deletes use filter : %d\n", FLAGS_filter_deletes);