diff --git a/include/leveldb/options.h b/include/leveldb/options.h index 5b37f4e50..28aaba485 100644 --- a/include/leveldb/options.h +++ b/include/leveldb/options.h @@ -345,6 +345,15 @@ struct Options { void Dump(Logger* log) const; + // Set appropriate parameters for bulk loading. + // The reason that this is a function that returns "this" instead of a + // constructure is to enable chaining of multiple similar calls in the future. + // + // All data will be in level 0 without any automatic compaction. + // It's recommended to manually call CompactRange(NULL, NULL) before reading + // from the database, because otherwise the read can be very slow. + Options* PrepareForBulkLoad(); + // This method allows an application to modify/delete a key-value at // the time of compaction. The compaction process invokes this // method for every kv that is being compacted. A return value diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index cf791e7ed..3553527cd 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -324,17 +324,23 @@ void CompactorCommand::DoCommand() { } const string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal"; +const string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load"; +const string DBLoaderCommand::ARG_COMPACT = "compact"; DBLoaderCommand::DBLoaderCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, false, BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO, ARG_CREATE_IF_MISSING, - ARG_DISABLE_WAL})), - create_if_missing_(false) { + ARG_DISABLE_WAL, ARG_BULK_LOAD, + ARG_COMPACT})), + create_if_missing_(false), disable_wal_(false), bulk_load_(false), + compact_(false) { create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING); disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL); + bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD); + compact_ = IsFlagPresent(flags, ARG_COMPACT); } void DBLoaderCommand::Help(string& ret) { @@ -342,12 +348,17 @@ void DBLoaderCommand::Help(string& ret) { ret.append(DBLoaderCommand::Name()); ret.append(" [--" + ARG_CREATE_IF_MISSING + "]"); ret.append(" [--" + ARG_DISABLE_WAL + "]"); + ret.append(" [--" + ARG_BULK_LOAD + "]"); + ret.append(" [--" + ARG_COMPACT + "]"); ret.append("\n"); } leveldb::Options DBLoaderCommand::PrepareOptionsForOpenDB() { leveldb::Options opt = LDBCommand::PrepareOptionsForOpenDB(); opt.create_if_missing = create_if_missing_; + if (bulk_load_) { + opt.PrepareForBulkLoad(); + } return opt; } @@ -380,6 +391,9 @@ void DBLoaderCommand::DoCommand() { if (bad_lines > 0) { std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl; } + if (compact_) { + db_->CompactRange(NULL, NULL); + } } const string DBDumperCommand::ARG_COUNT_ONLY = "count_only"; diff --git a/util/ldb_cmd.h b/util/ldb_cmd.h index 8aba628ee..210c78f46 100644 --- a/util/ldb_cmd.h +++ b/util/ldb_cmd.h @@ -372,8 +372,12 @@ public: private: bool create_if_missing_; bool disable_wal_; + bool bulk_load_; + bool compact_; static const string ARG_DISABLE_WAL; + static const string ARG_BULK_LOAD; + static const string ARG_COMPACT; }; class ReduceDBLevelsCommand : public LDBCommand { diff --git a/util/options.cc b/util/options.cc index 8adcde9c9..9597b7c85 100644 --- a/util/options.cc +++ b/util/options.cc @@ -161,4 +161,18 @@ Options::Dump(Logger* log) const manifest_preallocation_size); } // Options::Dump +Options* +Options::PrepareForBulkLoad() +{ + level0_file_num_compaction_trigger = (1<<30); + level0_slowdown_writes_trigger = (1<<30); + level0_stop_writes_trigger = (1<<30); + disable_auto_compactions = true; + disable_seek_compaction = true; + disableDataSync = true; + source_compaction_factor = (1<<30); + + return this; +} + } // namespace leveldb