andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Log info about generated blob files in BlobFileBuilder (#7324)

Browse Source 
Summary:
The patch adds a log message to `BlobFileBuilder` that is logged upon
generating a blob file, similarly to how we log the generation of table files
during flush and compaction. The log message contains the column family
name, job id, blob file number, and the number and total size of blobs in
the new file.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7324

Test Plan: Ran `make check` and checked the actual log messages using a custom `db_bench`.

Reviewed By: riversand963

Differential Revision: D23402229

Pulled By: ltamasi

fbshipit-source-id: ca42beb4db284b783d1eb2651f321032a45d0c5f
This commit is contained in:
Levi Tamasi 2020-08-31 12:30:02 -07:00 committed by Facebook GitHub Bot
parent 963314ffd6
commit 792d2f906e
3 changed files with 65 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
db/blob/blob_file_builder.cc
View File

@ -15,6 +15,7 @@
#include "file/filename.h" #include "file/filename.h"
#include "file/read_write_util.h" #include "file/read_write_util.h"
#include "file/writable_file_writer.h" #include "file/writable_file_writer.h"
#include "logging/logging.h"
#include "options/cf_options.h" #include "options/cf_options.h"
#include "rocksdb/slice.h" #include "rocksdb/slice.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
@ -27,19 +28,22 @@ BlobFileBuilder::BlobFileBuilder(
VersionSet* versions, Env* env, FileSystem* fs, VersionSet* versions, Env* env, FileSystem* fs,
const ImmutableCFOptions* immutable_cf_options, const ImmutableCFOptions* immutable_cf_options,
const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
uint32_t column_family_id, Env::IOPriority io_priority, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
std::vector<BlobFileAddition>* blob_file_additions) std::vector<BlobFileAddition>* blob_file_additions)
: BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env, : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env,
fs, immutable_cf_options, mutable_cf_options, fs, immutable_cf_options, mutable_cf_options,
file_options, column_family_id, io_priority, write_hint, file_options, job_id, column_family_id,
column_family_name, io_priority, write_hint,
blob_file_additions) {} blob_file_additions) {}
BlobFileBuilder::BlobFileBuilder( BlobFileBuilder::BlobFileBuilder(
std::function<uint64_t()> file_number_generator, Env* env, FileSystem* fs, std::function<uint64_t()> file_number_generator, Env* env, FileSystem* fs,
const ImmutableCFOptions* immutable_cf_options, const ImmutableCFOptions* immutable_cf_options,
const MutableCFOptions* mutable_cf_options, const FileOptions* file_options, const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
uint32_t column_family_id, Env::IOPriority io_priority, int job_id, uint32_t column_family_id,
const std::string& column_family_name, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
std::vector<BlobFileAddition>* blob_file_additions) std::vector<BlobFileAddition>* blob_file_additions)
: file_number_generator_(std::move(file_number_generator)), : file_number_generator_(std::move(file_number_generator)),
@ -50,7 +54,9 @@ BlobFileBuilder::BlobFileBuilder(
blob_file_size_(mutable_cf_options->blob_file_size), blob_file_size_(mutable_cf_options->blob_file_size),
blob_compression_type_(mutable_cf_options->blob_compression_type), blob_compression_type_(mutable_cf_options->blob_compression_type),
file_options_(file_options), file_options_(file_options),
job_id_(job_id),
column_family_id_(column_family_id), column_family_id_(column_family_id),
column_family_name_(column_family_name),
io_priority_(io_priority), io_priority_(io_priority),
write_hint_(write_hint), write_hint_(write_hint),
blob_file_additions_(blob_file_additions), blob_file_additions_(blob_file_additions),
@ -268,6 +274,13 @@ Status BlobFileBuilder::CloseBlobFile() {
std::move(checksum_method), std::move(checksum_method),
std::move(checksum_value)); std::move(checksum_value));
assert(immutable_cf_options_);
ROCKS_LOG_INFO(immutable_cf_options_->info_log,
"[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64
" total blobs, %" PRIu64 " total bytes",
column_family_name_.c_str(), job_id_, blob_file_number,
blob_count_, blob_bytes_);
writer_.reset(); writer_.reset();
blob_count_ = 0; blob_count_ = 0;
blob_bytes_ = 0; blob_bytes_ = 0;

10
db/blob/blob_file_builder.h
View File

@ -31,7 +31,9 @@ class BlobFileBuilder {
BlobFileBuilder(VersionSet* versions, Env* env, FileSystem* fs, BlobFileBuilder(VersionSet* versions, Env* env, FileSystem* fs,
const ImmutableCFOptions* immutable_cf_options, const ImmutableCFOptions* immutable_cf_options,
const MutableCFOptions* mutable_cf_options, const MutableCFOptions* mutable_cf_options,
const FileOptions* file_options, uint32_t column_family_id, const FileOptions* file_options, int job_id,
uint32_t column_family_id,
const std::string& column_family_name,
Env::IOPriority io_priority, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
std::vector<BlobFileAddition>* blob_file_additions); std::vector<BlobFileAddition>* blob_file_additions);
@ -40,7 +42,9 @@ class BlobFileBuilder {
FileSystem* fs, FileSystem* fs,
const ImmutableCFOptions* immutable_cf_options, const ImmutableCFOptions* immutable_cf_options,
const MutableCFOptions* mutable_cf_options, const MutableCFOptions* mutable_cf_options,
const FileOptions* file_options, uint32_t column_family_id, const FileOptions* file_options, int job_id,
uint32_t column_family_id,
const std::string& column_family_name,
Env::IOPriority io_priority, Env::IOPriority io_priority,
Env::WriteLifeTimeHint write_hint, Env::WriteLifeTimeHint write_hint,
std::vector<BlobFileAddition>* blob_file_additions); std::vector<BlobFileAddition>* blob_file_additions);
@ -70,7 +74,9 @@ class BlobFileBuilder {
uint64_t blob_file_size_; uint64_t blob_file_size_;
CompressionType blob_compression_type_; CompressionType blob_compression_type_;
const FileOptions* file_options_; const FileOptions* file_options_;
int job_id_;
uint32_t column_family_id_; uint32_t column_family_id_;
std::string column_family_name_;
Env::IOPriority io_priority_; Env::IOPriority io_priority_;
Env::WriteLifeTimeHint write_hint_; Env::WriteLifeTimeHint write_hint_;
std::vector<BlobFileAddition>* blob_file_additions_; std::vector<BlobFileAddition>* blob_file_additions_;

67
db/blob/blob_file_builder_test.cc
View File

@ -131,16 +131,18 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options,
&file_options_, column_family_id, io_priority, &mutable_cf_options, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint, &blob_file_additions);
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs( std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
number_of_blobs); number_of_blobs);
@ -202,16 +204,18 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options,
&file_options_, column_family_id, io_priority, &mutable_cf_options, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint, &blob_file_additions);
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs( std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
number_of_blobs); number_of_blobs);
@ -276,16 +280,18 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options,
&file_options_, column_family_id, io_priority, &mutable_cf_options, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint, &blob_file_additions);
for (size_t i = 0; i < number_of_blobs; ++i) { for (size_t i = 0; i < number_of_blobs; ++i) {
const std::string key = std::to_string(i); const std::string key = std::to_string(i);
@ -323,16 +329,18 @@ TEST_F(BlobFileBuilderTest, Compression) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options,
&file_options_, column_family_id, io_priority, &mutable_cf_options, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint, &blob_file_additions);
const std::string key("1"); const std::string key("1");
const std::string uncompressed_value(value_size, 'x'); const std::string uncompressed_value(value_size, 'x');
@ -393,16 +401,18 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options,
&file_options_, column_family_id, io_priority, &mutable_cf_options, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint, &blob_file_additions);
SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue", SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
[](void* arg) { [](void* arg) {
@ -457,16 +467,18 @@ TEST_F(BlobFileBuilderTest, Checksum) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(
&immutable_cf_options, &mutable_cf_options, TestFileNumberGenerator(), &mock_env_, &fs_, &immutable_cf_options,
&file_options_, column_family_id, io_priority, &mutable_cf_options, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint, &blob_file_additions);
const std::string key("1"); const std::string key("1");
const std::string value("deadbeef"); const std::string value("deadbeef");
@ -543,7 +555,9 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
ImmutableCFOptions immutable_cf_options(options); ImmutableCFOptions immutable_cf_options(options);
MutableCFOptions mutable_cf_options(options); MutableCFOptions mutable_cf_options(options);
constexpr int job_id = 1;
constexpr uint32_t column_family_id = 123; constexpr uint32_t column_family_id = 123;
constexpr char column_family_name[] = "foobar";
constexpr Env::IOPriority io_priority = Env::IO_HIGH; constexpr Env::IOPriority io_priority = Env::IO_HIGH;
constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM; constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
@ -551,8 +565,9 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
BlobFileBuilder builder(TestFileNumberGenerator(), &fault_injection_env_, BlobFileBuilder builder(TestFileNumberGenerator(), &fault_injection_env_,
&fs_, &immutable_cf_options, &mutable_cf_options, &fs_, &immutable_cf_options, &mutable_cf_options,
&file_options_, column_family_id, io_priority, &file_options_, job_id, column_family_id,
write_hint, &blob_file_additions); column_family_name, io_priority, write_hint,
&blob_file_additions);
SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) { SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
fault_injection_env_.SetFilesystemActive(false, fault_injection_env_.SetFilesystemActive(false,