Add support for IOTracing in blob files (#7958)
Summary: Add support for IOTracing in blob files Pull Request resolved: https://github.com/facebook/rocksdb/pull/7958 Test Plan: Add a new test and checked manually the trace_file for blob files being recorded during read and write. Reviewed By: ltamasi Differential Revision: D26415950 Pulled By: akankshamahajan15 fbshipit-source-id: 49c2859b3a4f8307e7cb69a92704403a4da46d44
This commit is contained in:
parent
9df78a94f1
commit
ea8bb82fc7
@ -21,6 +21,7 @@
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "test_util/sync_point.h"
|
||||
#include "trace_replay/io_tracer.h"
|
||||
#include "util/compression.h"
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
@ -32,12 +33,13 @@ BlobFileBuilder::BlobFileBuilder(
|
||||
int job_id, uint32_t column_family_id,
|
||||
const std::string& column_family_name, Env::IOPriority io_priority,
|
||||
Env::WriteLifeTimeHint write_hint,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
std::vector<std::string>* blob_file_paths,
|
||||
std::vector<BlobFileAddition>* blob_file_additions)
|
||||
: BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, env,
|
||||
fs, immutable_cf_options, mutable_cf_options,
|
||||
file_options, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
column_family_name, io_priority, write_hint, io_tracer,
|
||||
blob_file_paths, blob_file_additions) {}
|
||||
|
||||
BlobFileBuilder::BlobFileBuilder(
|
||||
@ -47,6 +49,7 @@ BlobFileBuilder::BlobFileBuilder(
|
||||
int job_id, uint32_t column_family_id,
|
||||
const std::string& column_family_name, Env::IOPriority io_priority,
|
||||
Env::WriteLifeTimeHint write_hint,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
std::vector<std::string>* blob_file_paths,
|
||||
std::vector<BlobFileAddition>* blob_file_additions)
|
||||
: file_number_generator_(std::move(file_number_generator)),
|
||||
@ -61,6 +64,7 @@ BlobFileBuilder::BlobFileBuilder(
|
||||
column_family_name_(column_family_name),
|
||||
io_priority_(io_priority),
|
||||
write_hint_(write_hint),
|
||||
io_tracer_(io_tracer),
|
||||
blob_file_paths_(blob_file_paths),
|
||||
blob_file_additions_(blob_file_additions),
|
||||
blob_count_(0),
|
||||
@ -182,7 +186,7 @@ Status BlobFileBuilder::OpenBlobFileIfNeeded() {
|
||||
Statistics* const statistics = immutable_cf_options_->statistics;
|
||||
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
|
||||
std::move(file), blob_file_paths_->back(), *file_options_, clock_,
|
||||
nullptr /*IOTracer*/, statistics, immutable_cf_options_->listeners,
|
||||
io_tracer_, statistics, immutable_cf_options_->listeners,
|
||||
immutable_cf_options_->file_checksum_gen_factory,
|
||||
tmp_set.Contains(FileType::kBlobFile)));
|
||||
|
||||
|
@ -26,6 +26,7 @@ class BlobFileAddition;
|
||||
class Status;
|
||||
class Slice;
|
||||
class BlobLogWriter;
|
||||
class IOTracer;
|
||||
|
||||
class BlobFileBuilder {
|
||||
public:
|
||||
@ -37,6 +38,7 @@ class BlobFileBuilder {
|
||||
const std::string& column_family_name,
|
||||
Env::IOPriority io_priority,
|
||||
Env::WriteLifeTimeHint write_hint,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
std::vector<std::string>* blob_file_paths,
|
||||
std::vector<BlobFileAddition>* blob_file_additions);
|
||||
|
||||
@ -49,6 +51,7 @@ class BlobFileBuilder {
|
||||
const std::string& column_family_name,
|
||||
Env::IOPriority io_priority,
|
||||
Env::WriteLifeTimeHint write_hint,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
std::vector<std::string>* blob_file_paths,
|
||||
std::vector<BlobFileAddition>* blob_file_additions);
|
||||
|
||||
@ -82,6 +85,7 @@ class BlobFileBuilder {
|
||||
std::string column_family_name_;
|
||||
Env::IOPriority io_priority_;
|
||||
Env::WriteLifeTimeHint write_hint_;
|
||||
std::shared_ptr<IOTracer> io_tracer_;
|
||||
std::vector<std::string>* blob_file_paths_;
|
||||
std::vector<BlobFileAddition>* blob_file_additions_;
|
||||
std::unique_ptr<BlobLogWriter> writer_;
|
||||
|
@ -140,10 +140,10 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
|
||||
@ -223,10 +223,10 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
|
||||
@ -308,10 +308,10 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
for (size_t i = 0; i < number_of_blobs; ++i) {
|
||||
@ -360,10 +360,10 @@ TEST_F(BlobFileBuilderTest, Compression) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
const std::string key("1");
|
||||
@ -442,10 +442,10 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
|
||||
@ -519,10 +519,10 @@ TEST_F(BlobFileBuilderTest, Checksum) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
const std::string key("1");
|
||||
@ -615,10 +615,10 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
|
||||
std::vector<std::string> blob_file_paths;
|
||||
std::vector<BlobFileAddition> blob_file_additions;
|
||||
|
||||
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_,
|
||||
&immutable_cf_options, &mutable_cf_options,
|
||||
&file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint,
|
||||
BlobFileBuilder builder(
|
||||
TestFileNumberGenerator(), &mock_env_, fs_, &immutable_cf_options,
|
||||
&mutable_cf_options, &file_options_, job_id, column_family_id,
|
||||
column_family_name, io_priority, write_hint, nullptr /*IOTracer*/,
|
||||
&blob_file_paths, &blob_file_additions);
|
||||
|
||||
SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
|
||||
|
@ -13,6 +13,7 @@
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/slice.h"
|
||||
#include "test_util/sync_point.h"
|
||||
#include "trace_replay/io_tracer.h"
|
||||
#include "util/hash.h"
|
||||
|
||||
namespace ROCKSDB_NAMESPACE {
|
||||
@ -21,13 +22,15 @@ BlobFileCache::BlobFileCache(Cache* cache,
|
||||
const ImmutableCFOptions* immutable_cf_options,
|
||||
const FileOptions* file_options,
|
||||
uint32_t column_family_id,
|
||||
HistogramImpl* blob_file_read_hist)
|
||||
HistogramImpl* blob_file_read_hist,
|
||||
const std::shared_ptr<IOTracer>& io_tracer)
|
||||
: cache_(cache),
|
||||
mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
|
||||
immutable_cf_options_(immutable_cf_options),
|
||||
file_options_(file_options),
|
||||
column_family_id_(column_family_id),
|
||||
blob_file_read_hist_(blob_file_read_hist) {
|
||||
blob_file_read_hist_(blob_file_read_hist),
|
||||
io_tracer_(io_tracer) {
|
||||
assert(cache_);
|
||||
assert(immutable_cf_options_);
|
||||
assert(file_options_);
|
||||
@ -71,7 +74,7 @@ Status BlobFileCache::GetBlobFileReader(
|
||||
assert(file_options_);
|
||||
const Status s = BlobFileReader::Create(
|
||||
*immutable_cf_options_, *file_options_, column_family_id_,
|
||||
blob_file_read_hist_, blob_file_number, &reader);
|
||||
blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
|
||||
if (!s.ok()) {
|
||||
RecordTick(statistics, NO_FILE_ERRORS);
|
||||
return s;
|
||||
|
@ -20,12 +20,14 @@ class HistogramImpl;
|
||||
class Status;
|
||||
class BlobFileReader;
|
||||
class Slice;
|
||||
class IOTracer;
|
||||
|
||||
class BlobFileCache {
|
||||
public:
|
||||
BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options,
|
||||
const FileOptions* file_options, uint32_t column_family_id,
|
||||
HistogramImpl* blob_file_read_hist);
|
||||
HistogramImpl* blob_file_read_hist,
|
||||
const std::shared_ptr<IOTracer>& io_tracer);
|
||||
|
||||
BlobFileCache(const BlobFileCache&) = delete;
|
||||
BlobFileCache& operator=(const BlobFileCache&) = delete;
|
||||
@ -42,6 +44,7 @@ class BlobFileCache {
|
||||
const FileOptions* file_options_;
|
||||
uint32_t column_family_id_;
|
||||
HistogramImpl* blob_file_read_hist_;
|
||||
std::shared_ptr<IOTracer> io_tracer_;
|
||||
|
||||
static constexpr size_t kNumberOfMutexStripes = 1 << 7;
|
||||
};
|
||||
|
@ -114,7 +114,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader) {
|
||||
|
||||
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
|
||||
&file_options, column_family_id,
|
||||
blob_file_read_hist);
|
||||
blob_file_read_hist, nullptr /*IOTracer*/);
|
||||
|
||||
// First try: reader should be opened and put in cache
|
||||
CacheHandleGuard<BlobFileReader> first;
|
||||
@ -159,7 +159,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
|
||||
|
||||
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
|
||||
&file_options, column_family_id,
|
||||
blob_file_read_hist);
|
||||
blob_file_read_hist, nullptr /*IOTracer*/);
|
||||
|
||||
CacheHandleGuard<BlobFileReader> first;
|
||||
CacheHandleGuard<BlobFileReader> second;
|
||||
@ -207,7 +207,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
|
||||
|
||||
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
|
||||
&file_options, column_family_id,
|
||||
blob_file_read_hist);
|
||||
blob_file_read_hist, nullptr /*IOTracer*/);
|
||||
|
||||
// Note: there is no blob file with the below number
|
||||
constexpr uint64_t blob_file_number = 123;
|
||||
@ -248,7 +248,7 @@ TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
|
||||
|
||||
BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
|
||||
&file_options, column_family_id,
|
||||
blob_file_read_hist);
|
||||
blob_file_read_hist, nullptr /*IOTracer*/);
|
||||
|
||||
// Insert into cache should fail since it has zero capacity and
|
||||
// strict_capacity_limit is set
|
||||
|
@ -24,6 +24,7 @@ Status BlobFileReader::Create(
|
||||
const ImmutableCFOptions& immutable_cf_options,
|
||||
const FileOptions& file_options, uint32_t column_family_id,
|
||||
HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
std::unique_ptr<BlobFileReader>* blob_file_reader) {
|
||||
assert(blob_file_reader);
|
||||
assert(!*blob_file_reader);
|
||||
@ -34,7 +35,7 @@ Status BlobFileReader::Create(
|
||||
{
|
||||
const Status s =
|
||||
OpenFile(immutable_cf_options, file_options, blob_file_read_hist,
|
||||
blob_file_number, &file_size, &file_reader);
|
||||
blob_file_number, io_tracer, &file_size, &file_reader);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
@ -68,8 +69,8 @@ Status BlobFileReader::Create(
|
||||
Status BlobFileReader::OpenFile(
|
||||
const ImmutableCFOptions& immutable_cf_options,
|
||||
const FileOptions& file_opts, HistogramImpl* blob_file_read_hist,
|
||||
uint64_t blob_file_number, uint64_t* file_size,
|
||||
std::unique_ptr<RandomAccessFileReader>* file_reader) {
|
||||
uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
|
||||
uint64_t* file_size, std::unique_ptr<RandomAccessFileReader>* file_reader) {
|
||||
assert(file_size);
|
||||
assert(file_reader);
|
||||
|
||||
@ -118,7 +119,7 @@ Status BlobFileReader::OpenFile(
|
||||
|
||||
file_reader->reset(new RandomAccessFileReader(
|
||||
std::move(file), blob_file_path,
|
||||
immutable_cf_options.env->GetSystemClock(), std::shared_ptr<IOTracer>(),
|
||||
immutable_cf_options.env->GetSystemClock(), io_tracer,
|
||||
immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS,
|
||||
blob_file_read_hist, immutable_cf_options.rate_limiter,
|
||||
immutable_cf_options.listeners));
|
||||
|
@ -29,6 +29,7 @@ class BlobFileReader {
|
||||
uint32_t column_family_id,
|
||||
HistogramImpl* blob_file_read_hist,
|
||||
uint64_t blob_file_number,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
std::unique_ptr<BlobFileReader>* reader);
|
||||
|
||||
BlobFileReader(const BlobFileReader&) = delete;
|
||||
@ -47,7 +48,9 @@ class BlobFileReader {
|
||||
static Status OpenFile(const ImmutableCFOptions& immutable_cf_options,
|
||||
const FileOptions& file_opts,
|
||||
HistogramImpl* blob_file_read_hist,
|
||||
uint64_t blob_file_number, uint64_t* file_size,
|
||||
uint64_t blob_file_number,
|
||||
const std::shared_ptr<IOTracer>& io_tracer,
|
||||
uint64_t* file_size,
|
||||
std::unique_ptr<RandomAccessFileReader>* file_reader);
|
||||
|
||||
static Status ReadHeader(const RandomAccessFileReader* file_reader,
|
||||
|
@ -142,9 +142,9 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
|
||||
|
||||
std::unique_ptr<BlobFileReader> reader;
|
||||
|
||||
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader));
|
||||
ASSERT_OK(BlobFileReader::Create(
|
||||
immutable_cf_options, FileOptions(), column_family_id,
|
||||
blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
|
||||
|
||||
// Make sure the blob can be retrieved with and without checksum verification
|
||||
ReadOptions read_options;
|
||||
@ -282,7 +282,8 @@ TEST_F(BlobFileReaderTest, Malformed) {
|
||||
|
||||
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader)
|
||||
blob_file_number, nullptr /*IOTracer*/,
|
||||
&reader)
|
||||
.IsCorruption());
|
||||
}
|
||||
|
||||
@ -315,7 +316,8 @@ TEST_F(BlobFileReaderTest, TTL) {
|
||||
|
||||
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader)
|
||||
blob_file_number, nullptr /*IOTracer*/,
|
||||
&reader)
|
||||
.IsCorruption());
|
||||
}
|
||||
|
||||
@ -353,7 +355,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
|
||||
|
||||
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader)
|
||||
blob_file_number, nullptr /*IOTracer*/,
|
||||
&reader)
|
||||
.IsCorruption());
|
||||
}
|
||||
|
||||
@ -391,7 +394,8 @@ TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
|
||||
|
||||
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader)
|
||||
blob_file_number, nullptr /*IOTracer*/,
|
||||
&reader)
|
||||
.IsCorruption());
|
||||
}
|
||||
|
||||
@ -429,7 +433,7 @@ TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
|
||||
ASSERT_TRUE(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
incorrect_column_family_id,
|
||||
blob_file_read_hist, blob_file_number,
|
||||
&reader)
|
||||
nullptr /*IOTracer*/, &reader)
|
||||
.IsCorruption());
|
||||
}
|
||||
|
||||
@ -460,9 +464,9 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {
|
||||
|
||||
std::unique_ptr<BlobFileReader> reader;
|
||||
|
||||
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader));
|
||||
ASSERT_OK(BlobFileReader::Create(
|
||||
immutable_cf_options, FileOptions(), column_family_id,
|
||||
blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
|
||||
|
||||
SyncPoint::GetInstance()->SetCallBack(
|
||||
"BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) {
|
||||
@ -516,9 +520,9 @@ TEST_F(BlobFileReaderTest, Compression) {
|
||||
|
||||
std::unique_ptr<BlobFileReader> reader;
|
||||
|
||||
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader));
|
||||
ASSERT_OK(BlobFileReader::Create(
|
||||
immutable_cf_options, FileOptions(), column_family_id,
|
||||
blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
|
||||
|
||||
// Make sure the blob can be retrieved with and without checksum verification
|
||||
ReadOptions read_options;
|
||||
@ -576,9 +580,9 @@ TEST_F(BlobFileReaderTest, UncompressionError) {
|
||||
|
||||
std::unique_ptr<BlobFileReader> reader;
|
||||
|
||||
ASSERT_OK(BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader));
|
||||
ASSERT_OK(BlobFileReader::Create(
|
||||
immutable_cf_options, FileOptions(), column_family_id,
|
||||
blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader));
|
||||
|
||||
SyncPoint::GetInstance()->SetCallBack(
|
||||
"BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) {
|
||||
@ -661,9 +665,9 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
|
||||
|
||||
std::unique_ptr<BlobFileReader> reader;
|
||||
|
||||
const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader);
|
||||
const Status s = BlobFileReader::Create(
|
||||
immutable_cf_options, FileOptions(), column_family_id,
|
||||
blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
|
||||
|
||||
const bool fail_during_create =
|
||||
(sync_point_ != "BlobFileReader::GetBlob:ReadFromFile");
|
||||
@ -741,9 +745,9 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
|
||||
|
||||
std::unique_ptr<BlobFileReader> reader;
|
||||
|
||||
const Status s = BlobFileReader::Create(immutable_cf_options, FileOptions(),
|
||||
column_family_id, blob_file_read_hist,
|
||||
blob_file_number, &reader);
|
||||
const Status s = BlobFileReader::Create(
|
||||
immutable_cf_options, FileOptions(), column_family_id,
|
||||
blob_file_read_hist, blob_file_number, nullptr /*IOTracer*/, &reader);
|
||||
|
||||
const bool fail_during_create =
|
||||
sync_point_ != "BlobFileReader::GetBlob:TamperWithResult";
|
||||
|
@ -210,6 +210,63 @@ TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
|
||||
.IsCorruption());
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
TEST_F(DBBlobBasicTest, GenerateIOTracing) {
|
||||
Options options = GetDefaultOptions();
|
||||
options.enable_blob_files = true;
|
||||
options.min_blob_size = 0;
|
||||
std::string trace_file = dbname_ + "/io_trace_file";
|
||||
|
||||
Reopen(options);
|
||||
{
|
||||
// Create IO trace file
|
||||
std::unique_ptr<TraceWriter> trace_writer;
|
||||
ASSERT_OK(
|
||||
NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer));
|
||||
ASSERT_OK(db_->StartIOTrace(env_, TraceOptions(), std::move(trace_writer)));
|
||||
|
||||
constexpr char key[] = "key";
|
||||
constexpr char blob_value[] = "blob_value";
|
||||
|
||||
ASSERT_OK(Put(key, blob_value));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_EQ(Get(key), blob_value);
|
||||
|
||||
ASSERT_OK(db_->EndIOTrace());
|
||||
ASSERT_OK(env_->FileExists(trace_file));
|
||||
}
|
||||
{
|
||||
// Parse trace file to check file opertions related to blob files are
|
||||
// recorded.
|
||||
std::unique_ptr<TraceReader> trace_reader;
|
||||
ASSERT_OK(
|
||||
NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader));
|
||||
IOTraceReader reader(std::move(trace_reader));
|
||||
|
||||
IOTraceHeader header;
|
||||
ASSERT_OK(reader.ReadHeader(&header));
|
||||
ASSERT_EQ(kMajorVersion, static_cast<int>(header.rocksdb_major_version));
|
||||
ASSERT_EQ(kMinorVersion, static_cast<int>(header.rocksdb_minor_version));
|
||||
|
||||
// Read records.
|
||||
int blob_files_op_count = 0;
|
||||
Status status;
|
||||
while (true) {
|
||||
IOTraceRecord record;
|
||||
status = reader.ReadIOOp(&record);
|
||||
if (!status.ok()) {
|
||||
break;
|
||||
}
|
||||
if (record.file_name.find("blob") != std::string::npos) {
|
||||
blob_files_op_count++;
|
||||
}
|
||||
}
|
||||
// Assuming blob files will have Append, Close and then Read operations.
|
||||
ASSERT_GT(blob_files_op_count, 2);
|
||||
}
|
||||
}
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
|
||||
public testing::WithParamInterface<std::string> {
|
||||
protected:
|
||||
|
@ -179,8 +179,8 @@ Status BuildTable(
|
||||
? new BlobFileBuilder(versions, env, fs, &ioptions,
|
||||
&mutable_cf_options, &file_options, job_id,
|
||||
column_family_id, column_family_name,
|
||||
io_priority, write_hint, &blob_file_paths,
|
||||
blob_file_additions)
|
||||
io_priority, write_hint, io_tracer,
|
||||
&blob_file_paths, blob_file_additions)
|
||||
: nullptr);
|
||||
|
||||
CompactionIterator c_iter(
|
||||
|
@ -562,7 +562,7 @@ ColumnFamilyData::ColumnFamilyData(
|
||||
block_cache_tracer, io_tracer));
|
||||
blob_file_cache_.reset(
|
||||
new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
|
||||
internal_stats_->GetBlobFileReadHist()));
|
||||
internal_stats_->GetBlobFileReadHist(), io_tracer));
|
||||
|
||||
if (ioptions_.compaction_style == kCompactionStyleLevel) {
|
||||
compaction_picker_.reset(
|
||||
|
@ -978,7 +978,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
|
||||
sub_compact->compaction->immutable_cf_options(),
|
||||
mutable_cf_options, &file_options_, job_id_, cfd->GetID(),
|
||||
cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_,
|
||||
&blob_file_paths, &sub_compact->blob_file_additions)
|
||||
io_tracer_, &blob_file_paths, &sub_compact->blob_file_additions)
|
||||
: nullptr);
|
||||
|
||||
TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
|
||||
|
Loading…
Reference in New Issue
Block a user