2019-09-16 19:31:27 +02:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
//
|
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
|
|
|
#include "file/random_access_file_reader.h"
|
|
|
|
|
|
|
|
#include <algorithm>
|
|
|
|
#include <mutex>
|
|
|
|
|
2021-01-26 07:07:26 +01:00
|
|
|
#include "file/file_util.h"
|
2019-09-16 19:31:27 +02:00
|
|
|
#include "monitoring/histogram.h"
|
|
|
|
#include "monitoring/iostats_context_imp.h"
|
|
|
|
#include "port/port.h"
|
2020-04-30 23:48:51 +02:00
|
|
|
#include "table/format.h"
|
2019-09-16 19:31:27 +02:00
|
|
|
#include "test_util/sync_point.h"
|
|
|
|
#include "util/random.h"
|
|
|
|
#include "util/rate_limiter.h"
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2021-01-29 07:08:46 +01:00
|
|
|
Status RandomAccessFileReader::Create(
|
|
|
|
const std::shared_ptr<FileSystem>& fs, const std::string& fname,
|
|
|
|
const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<RandomAccessFileReader>* reader, IODebugContext* dbg) {
|
|
|
|
std::unique_ptr<FSRandomAccessFile> file;
|
|
|
|
Status s = fs->NewRandomAccessFile(fname, file_opts, &file, dbg);
|
|
|
|
if (s.ok()) {
|
|
|
|
reader->reset(new RandomAccessFileReader(std::move(file), fname));
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
|
2020-04-30 23:48:51 +02:00
|
|
|
Status RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
|
|
|
|
size_t n, Slice* result, char* scratch,
|
|
|
|
AlignedBuf* aligned_buf,
|
2020-03-06 23:02:09 +01:00
|
|
|
bool for_compaction) const {
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
(void)aligned_buf;
|
2020-05-13 03:21:32 +02:00
|
|
|
|
|
|
|
TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read", nullptr);
|
2019-09-16 19:31:27 +02:00
|
|
|
Status s;
|
|
|
|
uint64_t elapsed = 0;
|
|
|
|
{
|
2021-01-26 07:07:26 +01:00
|
|
|
StopWatch sw(clock_, stats_, hist_type_,
|
2019-09-16 19:31:27 +02:00
|
|
|
(stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/,
|
|
|
|
true /*delay_enabled*/);
|
|
|
|
auto prev_perf_level = GetPerfLevel();
|
|
|
|
IOSTATS_TIMER_GUARD(read_nanos);
|
|
|
|
if (use_direct_io()) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
size_t alignment = file_->GetRequiredBufferAlignment();
|
|
|
|
size_t aligned_offset =
|
|
|
|
TruncateToPageBoundary(alignment, static_cast<size_t>(offset));
|
|
|
|
size_t offset_advance = static_cast<size_t>(offset) - aligned_offset;
|
|
|
|
size_t read_size =
|
|
|
|
Roundup(static_cast<size_t>(offset + n), alignment) - aligned_offset;
|
|
|
|
AlignedBuffer buf;
|
|
|
|
buf.Alignment(alignment);
|
|
|
|
buf.AllocateNewBuffer(read_size);
|
|
|
|
while (buf.CurrentSize() < read_size) {
|
|
|
|
size_t allowed;
|
|
|
|
if (for_compaction && rate_limiter_ != nullptr) {
|
|
|
|
allowed = rate_limiter_->RequestToken(
|
|
|
|
buf.Capacity() - buf.CurrentSize(), buf.Alignment(),
|
|
|
|
Env::IOPriority::IO_LOW, stats_, RateLimiter::OpType::kRead);
|
|
|
|
} else {
|
|
|
|
assert(buf.CurrentSize() == 0);
|
|
|
|
allowed = read_size;
|
|
|
|
}
|
|
|
|
Slice tmp;
|
|
|
|
|
2020-07-22 17:53:21 +02:00
|
|
|
FileOperationInfo::StartTimePoint start_ts;
|
2019-09-16 19:31:27 +02:00
|
|
|
uint64_t orig_offset = 0;
|
|
|
|
if (ShouldNotifyListeners()) {
|
2020-07-22 17:53:21 +02:00
|
|
|
start_ts = FileOperationInfo::StartNow();
|
2019-09-16 19:31:27 +02:00
|
|
|
orig_offset = aligned_offset + buf.CurrentSize();
|
|
|
|
}
|
2020-04-30 23:48:51 +02:00
|
|
|
|
2019-09-16 19:31:27 +02:00
|
|
|
{
|
2021-01-26 07:07:26 +01:00
|
|
|
IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_);
|
2020-04-30 23:48:51 +02:00
|
|
|
// Only user reads are expected to specify a timeout. And user reads
|
|
|
|
// are not subjected to rate_limiter and should go through only
|
|
|
|
// one iteration of this loop, so we don't need to check and adjust
|
|
|
|
// the opts.timeout before calling file_->Read
|
|
|
|
assert(!opts.timeout.count() || allowed == read_size);
|
|
|
|
s = file_->Read(aligned_offset + buf.CurrentSize(), allowed, opts,
|
|
|
|
&tmp, buf.Destination(), nullptr);
|
2019-09-16 19:31:27 +02:00
|
|
|
}
|
|
|
|
if (ShouldNotifyListeners()) {
|
2020-07-22 17:53:21 +02:00
|
|
|
auto finish_ts = FileOperationInfo::FinishNow();
|
2019-09-16 19:31:27 +02:00
|
|
|
NotifyOnFileReadFinish(orig_offset, tmp.size(), start_ts, finish_ts,
|
|
|
|
s);
|
|
|
|
}
|
|
|
|
|
|
|
|
buf.Size(buf.CurrentSize() + tmp.size());
|
|
|
|
if (!s.ok() || tmp.size() < allowed) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
size_t res_len = 0;
|
|
|
|
if (s.ok() && offset_advance < buf.CurrentSize()) {
|
2020-03-06 23:02:09 +01:00
|
|
|
res_len = std::min(buf.CurrentSize() - offset_advance, n);
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
if (aligned_buf == nullptr) {
|
2020-03-06 23:02:09 +01:00
|
|
|
buf.Read(scratch, offset_advance, res_len);
|
|
|
|
} else {
|
2020-04-09 06:17:42 +02:00
|
|
|
scratch = buf.BufferStart() + offset_advance;
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
aligned_buf->reset(buf.Release());
|
2020-03-06 23:02:09 +01:00
|
|
|
}
|
2019-09-16 19:31:27 +02:00
|
|
|
}
|
|
|
|
*result = Slice(scratch, res_len);
|
|
|
|
#endif // !ROCKSDB_LITE
|
|
|
|
} else {
|
|
|
|
size_t pos = 0;
|
|
|
|
const char* res_scratch = nullptr;
|
|
|
|
while (pos < n) {
|
|
|
|
size_t allowed;
|
|
|
|
if (for_compaction && rate_limiter_ != nullptr) {
|
|
|
|
if (rate_limiter_->IsRateLimited(RateLimiter::OpType::kRead)) {
|
|
|
|
sw.DelayStart();
|
|
|
|
}
|
|
|
|
allowed = rate_limiter_->RequestToken(n - pos, 0 /* alignment */,
|
|
|
|
Env::IOPriority::IO_LOW, stats_,
|
|
|
|
RateLimiter::OpType::kRead);
|
|
|
|
if (rate_limiter_->IsRateLimited(RateLimiter::OpType::kRead)) {
|
|
|
|
sw.DelayStop();
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
allowed = n;
|
|
|
|
}
|
|
|
|
Slice tmp_result;
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
2020-07-22 17:53:21 +02:00
|
|
|
FileOperationInfo::StartTimePoint start_ts;
|
2019-09-16 19:31:27 +02:00
|
|
|
if (ShouldNotifyListeners()) {
|
2020-07-22 17:53:21 +02:00
|
|
|
start_ts = FileOperationInfo::StartNow();
|
2019-09-16 19:31:27 +02:00
|
|
|
}
|
|
|
|
#endif
|
2020-04-30 23:48:51 +02:00
|
|
|
|
2019-09-16 19:31:27 +02:00
|
|
|
{
|
2021-01-26 07:07:26 +01:00
|
|
|
IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_);
|
2020-04-30 23:48:51 +02:00
|
|
|
// Only user reads are expected to specify a timeout. And user reads
|
|
|
|
// are not subjected to rate_limiter and should go through only
|
|
|
|
// one iteration of this loop, so we don't need to check and adjust
|
|
|
|
// the opts.timeout before calling file_->Read
|
|
|
|
assert(!opts.timeout.count() || allowed == n);
|
|
|
|
s = file_->Read(offset + pos, allowed, opts, &tmp_result,
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
2019-12-13 23:47:08 +01:00
|
|
|
scratch + pos, nullptr);
|
2019-09-16 19:31:27 +02:00
|
|
|
}
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (ShouldNotifyListeners()) {
|
2020-07-22 17:53:21 +02:00
|
|
|
auto finish_ts = FileOperationInfo::FinishNow();
|
2019-09-16 19:31:27 +02:00
|
|
|
NotifyOnFileReadFinish(offset + pos, tmp_result.size(), start_ts,
|
|
|
|
finish_ts, s);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (res_scratch == nullptr) {
|
|
|
|
// we can't simply use `scratch` because reads of mmap'd files return
|
|
|
|
// data in a different buffer.
|
|
|
|
res_scratch = tmp_result.data();
|
|
|
|
} else {
|
|
|
|
// make sure chunks are inserted contiguously into `res_scratch`.
|
|
|
|
assert(tmp_result.data() == res_scratch + pos);
|
|
|
|
}
|
|
|
|
pos += tmp_result.size();
|
|
|
|
if (!s.ok() || tmp_result.size() < allowed) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*result = Slice(res_scratch, s.ok() ? pos : 0);
|
|
|
|
}
|
|
|
|
IOSTATS_ADD_IF_POSITIVE(bytes_read, result->size());
|
|
|
|
SetPerfLevel(prev_perf_level);
|
|
|
|
}
|
|
|
|
if (stats_ != nullptr && file_read_hist_ != nullptr) {
|
|
|
|
file_read_hist_->Add(elapsed);
|
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
size_t End(const FSReadRequest& r) {
|
|
|
|
return static_cast<size_t>(r.offset) + r.len;
|
|
|
|
}
|
|
|
|
|
|
|
|
FSReadRequest Align(const FSReadRequest& r, size_t alignment) {
|
|
|
|
FSReadRequest req;
|
|
|
|
req.offset = static_cast<uint64_t>(
|
|
|
|
TruncateToPageBoundary(alignment, static_cast<size_t>(r.offset)));
|
|
|
|
req.len = Roundup(End(r), alignment) - req.offset;
|
2020-03-24 04:12:38 +01:00
|
|
|
req.scratch = nullptr;
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
return req;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool TryMerge(FSReadRequest* dest, const FSReadRequest& src) {
|
|
|
|
size_t dest_offset = static_cast<size_t>(dest->offset);
|
|
|
|
size_t src_offset = static_cast<size_t>(src.offset);
|
|
|
|
size_t dest_end = End(*dest);
|
|
|
|
size_t src_end = End(src);
|
2020-07-23 00:02:10 +02:00
|
|
|
if (std::max(dest_offset, src_offset) > std::min(dest_end, src_end)) {
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
dest->offset = static_cast<uint64_t>(std::min(dest_offset, src_offset));
|
|
|
|
dest->len = std::max(dest_end, src_end) - dest->offset;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-04-30 23:48:51 +02:00
|
|
|
Status RandomAccessFileReader::MultiRead(const IOOptions& opts,
|
|
|
|
FSReadRequest* read_reqs,
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
size_t num_reqs,
|
|
|
|
AlignedBuf* aligned_buf) const {
|
|
|
|
(void)aligned_buf; // suppress warning of unused variable in LITE mode
|
|
|
|
assert(num_reqs > 0);
|
2019-09-16 19:31:27 +02:00
|
|
|
Status s;
|
|
|
|
uint64_t elapsed = 0;
|
|
|
|
{
|
2021-01-26 07:07:26 +01:00
|
|
|
StopWatch sw(clock_, stats_, hist_type_,
|
2019-09-16 19:31:27 +02:00
|
|
|
(stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/,
|
|
|
|
true /*delay_enabled*/);
|
|
|
|
auto prev_perf_level = GetPerfLevel();
|
|
|
|
IOSTATS_TIMER_GUARD(read_nanos);
|
|
|
|
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
FSReadRequest* fs_reqs = read_reqs;
|
|
|
|
size_t num_fs_reqs = num_reqs;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
std::vector<FSReadRequest> aligned_reqs;
|
|
|
|
if (use_direct_io()) {
|
|
|
|
// num_reqs is the max possible size,
|
|
|
|
// this can reduce std::vecector's internal resize operations.
|
|
|
|
aligned_reqs.reserve(num_reqs);
|
|
|
|
// Align and merge the read requests.
|
|
|
|
size_t alignment = file_->GetRequiredBufferAlignment();
|
2020-12-23 00:08:17 +01:00
|
|
|
for (size_t i = 0; i < num_reqs; i++) {
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
const auto& r = Align(read_reqs[i], alignment);
|
2020-12-23 00:08:17 +01:00
|
|
|
if (i == 0) {
|
|
|
|
// head
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
aligned_reqs.push_back(r);
|
2020-12-23 00:08:17 +01:00
|
|
|
|
|
|
|
} else if (!TryMerge(&aligned_reqs.back(), r)) {
|
|
|
|
// head + n
|
|
|
|
aligned_reqs.push_back(r);
|
|
|
|
|
|
|
|
} else {
|
|
|
|
// unused
|
|
|
|
r.status.PermitUncheckedError();
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
}
|
|
|
|
}
|
2020-07-23 22:48:17 +02:00
|
|
|
TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::MultiRead:AlignedReqs",
|
|
|
|
&aligned_reqs);
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
|
|
|
|
// Allocate aligned buffer and let scratch buffers point to it.
|
|
|
|
size_t total_len = 0;
|
|
|
|
for (const auto& r : aligned_reqs) {
|
|
|
|
total_len += r.len;
|
|
|
|
}
|
|
|
|
AlignedBuffer buf;
|
|
|
|
buf.Alignment(alignment);
|
|
|
|
buf.AllocateNewBuffer(total_len);
|
|
|
|
char* scratch = buf.BufferStart();
|
|
|
|
for (auto& r : aligned_reqs) {
|
|
|
|
r.scratch = scratch;
|
|
|
|
scratch += r.len;
|
|
|
|
}
|
|
|
|
|
|
|
|
aligned_buf->reset(buf.Release());
|
|
|
|
fs_reqs = aligned_reqs.data();
|
|
|
|
num_fs_reqs = aligned_reqs.size();
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
2019-09-16 19:31:27 +02:00
|
|
|
#ifndef ROCKSDB_LITE
|
2020-07-22 17:53:21 +02:00
|
|
|
FileOperationInfo::StartTimePoint start_ts;
|
2019-09-16 19:31:27 +02:00
|
|
|
if (ShouldNotifyListeners()) {
|
2020-07-22 17:53:21 +02:00
|
|
|
start_ts = FileOperationInfo::StartNow();
|
2019-09-16 19:31:27 +02:00
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
2020-04-30 23:48:51 +02:00
|
|
|
|
2019-09-16 19:31:27 +02:00
|
|
|
{
|
2021-01-26 07:07:26 +01:00
|
|
|
IOSTATS_CPU_TIMER_GUARD(cpu_read_nanos, clock_);
|
2020-04-30 23:48:51 +02:00
|
|
|
s = file_->MultiRead(fs_reqs, num_fs_reqs, opts, nullptr);
|
2019-09-16 19:31:27 +02:00
|
|
|
}
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (use_direct_io()) {
|
|
|
|
// Populate results in the unaligned read requests.
|
|
|
|
size_t aligned_i = 0;
|
|
|
|
for (size_t i = 0; i < num_reqs; i++) {
|
|
|
|
auto& r = read_reqs[i];
|
|
|
|
if (static_cast<size_t>(r.offset) > End(aligned_reqs[aligned_i])) {
|
|
|
|
aligned_i++;
|
|
|
|
}
|
|
|
|
const auto& fs_r = fs_reqs[aligned_i];
|
|
|
|
r.status = fs_r.status;
|
|
|
|
if (r.status.ok()) {
|
|
|
|
uint64_t offset = r.offset - fs_r.offset;
|
|
|
|
size_t len = std::min(r.len, static_cast<size_t>(fs_r.len - offset));
|
|
|
|
r.result = Slice(fs_r.scratch + offset, len);
|
|
|
|
} else {
|
|
|
|
r.result = Slice();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
2019-09-16 19:31:27 +02:00
|
|
|
for (size_t i = 0; i < num_reqs; ++i) {
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
if (ShouldNotifyListeners()) {
|
2020-07-22 17:53:21 +02:00
|
|
|
auto finish_ts = FileOperationInfo::FinishNow();
|
2019-09-16 19:31:27 +02:00
|
|
|
NotifyOnFileReadFinish(read_reqs[i].offset, read_reqs[i].result.size(),
|
|
|
|
start_ts, finish_ts, read_reqs[i].status);
|
|
|
|
}
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
IOSTATS_ADD_IF_POSITIVE(bytes_read, read_reqs[i].result.size());
|
|
|
|
}
|
|
|
|
SetPerfLevel(prev_perf_level);
|
|
|
|
}
|
|
|
|
if (stats_ != nullptr && file_read_hist_ != nullptr) {
|
|
|
|
file_read_hist_->Add(elapsed);
|
|
|
|
}
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
Support direct IO in RandomAccessFileReader::MultiRead (#6446)
Summary:
By supporting direct IO in RandomAccessFileReader::MultiRead, the benefits of parallel IO (IO uring) and direct IO can be combined.
In direct IO mode, read requests are aligned and merged together before being issued to RandomAccessFile::MultiRead, so blocks in the original requests might share the same underlying buffer, the shared buffers are returned in `aligned_bufs`, which is a new parameter of the `MultiRead` API.
For example, suppose alignment requirement for direct IO is 4KB, one request is (offset: 1KB, len: 1KB), another request is (offset: 3KB, len: 1KB), then since they all belong to page (offset: 0, len: 4KB), `MultiRead` only reads the page with direct IO into a buffer on heap, and returns 2 Slices referencing regions in that same buffer. See `random_access_file_reader_test.cc` for more examples.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/6446
Test Plan: Added a new test `random_access_file_reader_test.cc`.
Reviewed By: anand1976
Differential Revision: D20097518
Pulled By: cheng-chang
fbshipit-source-id: ca48a8faf9c3af146465c102ef6b266a363e78d1
2020-03-21 00:15:40 +01:00
|
|
|
|
2021-01-26 07:07:26 +01:00
|
|
|
IOStatus RandomAccessFileReader::PrepareIOOptions(const ReadOptions& ro,
|
|
|
|
IOOptions& opts) {
|
|
|
|
if (clock_.get() != nullptr) {
|
|
|
|
return PrepareIOFromReadOptions(ro, clock_, opts);
|
|
|
|
} else {
|
|
|
|
return PrepareIOFromReadOptions(ro, SystemClock::Default(), opts);
|
|
|
|
}
|
|
|
|
}
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|