andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make StringEnv, StringSink, StringSource use FS classes (#7786)

Browse Source 
Summary:
Change the StringEnv and related classes to be based on FileSystem APIs rather than the corresponding Env ones.  The StringSink and StringSource classes were changed to be based on the corresponding FS file classes.

Part of a cleanup to use the newer interfaces.  This change also eliminates some of the casts/wrappers to LegacyFile classes.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7786

Reviewed By: jay-zhuang

Differential Revision: D25761460

Pulled By: anand1976

fbshipit-source-id: 428ae8e32b3db97dbeeca08c9d3bb0d9d4d3a38f
This commit is contained in:
mrambacher 2021-01-04 15:59:52 -08:00 committed by Facebook GitHub Bot
parent 58660bf21a
commit c1a65a4de4
12 changed files with 490 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
db/log_test.cc
View File

@ -9,7 +9,6 @@
#include "db/log_reader.h" #include "db/log_reader.h"
#include "db/log_writer.h" #include "db/log_writer.h"
#include "env/composite_env_wrapper.h"
#include "file/sequence_file_reader.h" #include "file/sequence_file_reader.h"
#include "file/writable_file_writer.h" #include "file/writable_file_writer.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
@ -50,7 +49,7 @@ static std::string RandomSkewedString(int i, Random* rnd) {
// get<1>(tuple): true if allow retry after read EOF, false otherwise // get<1>(tuple): true if allow retry after read EOF, false otherwise
class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> { class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
private: private:
class StringSource : public SequentialFile { class StringSource : public FSSequentialFile {
public: public:
Slice& contents_; Slice& contents_;
bool force_error_; bool force_error_;
@ -68,7 +67,8 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
returned_partial_(false), returned_partial_(false),
fail_after_read_partial_(fail_after_read_partial) {} fail_after_read_partial_(fail_after_read_partial) {}
Status Read(size_t n, Slice* result, char* scratch) override { IOStatus Read(size_t n, const IOOptions& /*opts*/, Slice* result,
char* scratch, IODebugContext* /*dbg*/) override {
if (fail_after_read_partial_) { if (fail_after_read_partial_) {
EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error"; EXPECT_TRUE(!returned_partial_) << "must not Read() after eof/error";
} }
@ -81,7 +81,7 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
contents_.remove_prefix(force_error_position_); contents_.remove_prefix(force_error_position_);
force_error_ = false; force_error_ = false;
returned_partial_ = true; returned_partial_ = true;
return Status::Corruption("read error"); return IOStatus::Corruption("read error");
} }
} }
@ -106,28 +106,21 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
*result = Slice(scratch, n); *result = Slice(scratch, n);
contents_.remove_prefix(n); contents_.remove_prefix(n);
return Status::OK(); return IOStatus::OK();
} }
Status Skip(uint64_t n) override { IOStatus Skip(uint64_t n) override {
if (n > contents_.size()) { if (n > contents_.size()) {
contents_.clear(); contents_.clear();
return Status::NotFound("in-memory file skipepd past end"); return IOStatus::NotFound("in-memory file skipepd past end");
} }
contents_.remove_prefix(n); contents_.remove_prefix(n);
return Status::OK(); return IOStatus::OK();
} }
}; };
inline StringSource* GetStringSourceFromLegacyReader(
SequentialFileReader* reader) {
LegacySequentialFileWrapper* file =
static_cast<LegacySequentialFileWrapper*>(reader->file());
return static_cast<StringSource*>(file->target());
}
class ReportCollector : public Reader::Reporter { class ReportCollector : public Reader::Reporter {
public: public:
size_t dropped_bytes_; size_t dropped_bytes_;
@ -140,29 +133,17 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
} }
}; };
std::string& dest_contents() { std::string& dest_contents() { return sink_->contents_; }
auto dest = test::GetStringSinkFromLegacyWriter(writer_.file());
assert(dest);
return dest->contents_;
}
const std::string& dest_contents() const { const std::string& dest_contents() const { return sink_->contents_; }
auto dest = test::GetStringSinkFromLegacyWriter(writer_.file());
assert(dest);
return dest->contents_;
}
void reset_source_contents() { void reset_source_contents() { source_->contents_ = dest_contents(); }
auto src = GetStringSourceFromLegacyReader(reader_->file());
assert(src);
src->contents_ = dest_contents();
}
Slice reader_contents_; Slice reader_contents_;
std::unique_ptr<WritableFileWriter> dest_holder_; test::StringSink* sink_;
std::unique_ptr<SequentialFileReader> source_holder_; StringSource* source_;
ReportCollector report_; ReportCollector report_;
Writer writer_; std::unique_ptr<Writer> writer_;
std::unique_ptr<Reader> reader_; std::unique_ptr<Reader> reader_;
protected: protected:
@ -171,19 +152,23 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
public: public:
LogTest() LogTest()
: reader_contents_(), : reader_contents_(),
dest_holder_(test::GetWritableFileWriter( sink_(new test::StringSink(&reader_contents_)),
new test::StringSink(&reader_contents_), "" /* don't care */)), source_(new StringSource(reader_contents_, !std::get<1>(GetParam()))),
source_holder_(test::GetSequentialFileReader(
new StringSource(reader_contents_, !std::get<1>(GetParam())),
"" /* file name */)),
writer_(std::move(dest_holder_), 123, std::get<0>(GetParam())),
allow_retry_read_(std::get<1>(GetParam())) { allow_retry_read_(std::get<1>(GetParam())) {
std::unique_ptr<FSWritableFile> sink_holder(sink_);
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(sink_holder), "" /* don't care */, FileOptions()));
writer_.reset(
new Writer(std::move(file_writer), 123, std::get<0>(GetParam())));
std::unique_ptr<FSSequentialFile> source_holder(source_);
std::unique_ptr<SequentialFileReader> file_reader(
new SequentialFileReader(std::move(source_holder), "" /* file name */));
if (allow_retry_read_) { if (allow_retry_read_) {
reader_.reset(new FragmentBufferedReader( reader_.reset(new FragmentBufferedReader(nullptr, std::move(file_reader),
nullptr, std::move(source_holder_), &report_, true /* checksum */, &report_, true /* checksum */,
123 /* log_number */)); 123 /* log_number */));
} else { } else {
reader_.reset(new Reader(nullptr, std::move(source_holder_), &report_, reader_.reset(new Reader(nullptr, std::move(file_reader), &report_,
true /* checksum */, 123 /* log_number */)); true /* checksum */, 123 /* log_number */));
} }
} }
@ -191,7 +176,7 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
Slice* get_reader_contents() { return &reader_contents_; } Slice* get_reader_contents() { return &reader_contents_; }
void Write(const std::string& msg) { void Write(const std::string& msg) {
ASSERT_OK(writer_.AddRecord(Slice(msg))); ASSERT_OK(writer_->AddRecord(Slice(msg)));
} }
size_t WrittenBytes() const { size_t WrittenBytes() const {
@ -219,11 +204,7 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
dest_contents()[offset] = new_byte; dest_contents()[offset] = new_byte;
} }
void ShrinkSize(int bytes) { void ShrinkSize(int bytes) { sink_->Drop(bytes); }
auto dest = test::GetStringSinkFromLegacyWriter(writer_.file());
assert(dest);
dest->Drop(bytes);
}
void FixChecksum(int header_offset, int len, bool recyclable) { void FixChecksum(int header_offset, int len, bool recyclable) {
// Compute crc of type/len/data // Compute crc of type/len/data
@ -235,9 +216,8 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
} }
void ForceError(size_t position = 0) { void ForceError(size_t position = 0) {
auto src = GetStringSourceFromLegacyReader(reader_->file()); source_->force_error_ = true;
src->force_error_ = true; source_->force_error_position_ = position;
src->force_error_position_ = position;
} }
size_t DroppedBytes() const { size_t DroppedBytes() const {
@ -249,14 +229,12 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
} }
void ForceEOF(size_t position = 0) { void ForceEOF(size_t position = 0) {
auto src = GetStringSourceFromLegacyReader(reader_->file()); source_->force_eof_ = true;
src->force_eof_ = true; source_->force_eof_position_ = position;
src->force_eof_position_ = position;
} }
void UnmarkEOF() { void UnmarkEOF() {
auto src = GetStringSourceFromLegacyReader(reader_->file()); source_->returned_partial_ = false;
src->returned_partial_ = false;
reader_->UnmarkEOF(); reader_->UnmarkEOF();
} }
@ -685,9 +663,10 @@ TEST_P(LogTest, Recycle) {
while (get_reader_contents()->size() < log::kBlockSize * 2) { while (get_reader_contents()->size() < log::kBlockSize * 2) {
Write("xxxxxxxxxxxxxxxx"); Write("xxxxxxxxxxxxxxxx");
} }
std::unique_ptr<WritableFileWriter> dest_holder(test::GetWritableFileWriter( std::unique_ptr<FSWritableFile> sink(
new test::OverwritingStringSink(get_reader_contents()), new test::OverwritingStringSink(get_reader_contents()));
"" /* don't care */)); std::unique_ptr<WritableFileWriter> dest_holder(new WritableFileWriter(
std::move(sink), "" /* don't care */, FileOptions()));
Writer recycle_writer(std::move(dest_holder), 123, true); Writer recycle_writer(std::move(dest_holder), 123, true);
ASSERT_OK(recycle_writer.AddRecord(Slice("foooo"))); ASSERT_OK(recycle_writer.AddRecord(Slice("foooo")));
ASSERT_OK(recycle_writer.AddRecord(Slice("bar"))); ASSERT_OK(recycle_writer.AddRecord(Slice("bar")));
@ -718,10 +697,9 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
}; };
Slice contents_; Slice contents_;
std::unique_ptr<WritableFileWriter> dest_holder_; test::StringSink* sink_;
std::unique_ptr<Writer> log_writer_; std::unique_ptr<Writer> log_writer_;
Env* env_; Env* env_;
EnvOptions env_options_;
const std::string test_dir_; const std::string test_dir_;
const std::string log_file_; const std::string log_file_;
std::unique_ptr<WritableFileWriter> writer_; std::unique_ptr<WritableFileWriter> writer_;
@ -732,55 +710,50 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
public: public:
RetriableLogTest() RetriableLogTest()
: contents_(), : contents_(),
dest_holder_(nullptr), sink_(new test::StringSink(&contents_)),
log_writer_(nullptr), log_writer_(nullptr),
env_(Env::Default()), env_(Env::Default()),
test_dir_(test::PerThreadDBPath("retriable_log_test")), test_dir_(test::PerThreadDBPath("retriable_log_test")),
log_file_(test_dir_ + "/log"), log_file_(test_dir_ + "/log"),
writer_(nullptr), writer_(nullptr),
reader_(nullptr), reader_(nullptr),
log_reader_(nullptr) {} log_reader_(nullptr) {
std::unique_ptr<FSWritableFile> sink_holder(sink_);
std::unique_ptr<WritableFileWriter> wfw(new WritableFileWriter(
std::move(sink_holder), "" /* file name */, FileOptions()));
log_writer_.reset(new Writer(std::move(wfw), 123, GetParam()));
}
Status SetupTestEnv() { Status SetupTestEnv() {
dest_holder_.reset(test::GetWritableFileWriter(
new test::StringSink(&contents_), "" /* file name */));
assert(dest_holder_ != nullptr);
log_writer_.reset(new Writer(std::move(dest_holder_), 123, GetParam()));
assert(log_writer_ != nullptr);
Status s; Status s;
s = env_->CreateDirIfMissing(test_dir_); FileOptions fopts;
std::unique_ptr<WritableFile> writable_file; auto fs = env_->GetFileSystem();
s = fs->CreateDirIfMissing(test_dir_, IOOptions(), nullptr);
std::unique_ptr<FSWritableFile> writable_file;
if (s.ok()) { if (s.ok()) {
s = env_->NewWritableFile(log_file_, &writable_file, env_options_); s = fs->NewWritableFile(log_file_, fopts, &writable_file, nullptr);
} }
if (s.ok()) { if (s.ok()) {
writer_.reset(new WritableFileWriter( writer_.reset(
NewLegacyWritableFileWrapper(std::move(writable_file)), log_file_, new WritableFileWriter(std::move(writable_file), log_file_, fopts));
env_options_)); EXPECT_NE(writer_, nullptr);
assert(writer_ != nullptr);
} }
std::unique_ptr<SequentialFile> seq_file; std::unique_ptr<FSSequentialFile> seq_file;
if (s.ok()) { if (s.ok()) {
s = env_->NewSequentialFile(log_file_, &seq_file, env_options_); s = fs->NewSequentialFile(log_file_, fopts, &seq_file, nullptr);
} }
if (s.ok()) { if (s.ok()) {
reader_.reset(new SequentialFileReader( reader_.reset(new SequentialFileReader(std::move(seq_file), log_file_));
NewLegacySequentialFileWrapper(seq_file), log_file_)); EXPECT_NE(reader_, nullptr);
assert(reader_ != nullptr);
log_reader_.reset(new FragmentBufferedReader( log_reader_.reset(new FragmentBufferedReader(
nullptr, std::move(reader_), &report_, true /* checksum */, nullptr, std::move(reader_), &report_, true /* checksum */,
123 /* log_number */)); 123 /* log_number */));
assert(log_reader_ != nullptr); EXPECT_NE(log_reader_, nullptr);
} }
return s; return s;
} }
std::string contents() { std::string contents() { return sink_->contents_; }
auto file = test::GetStringSinkFromLegacyWriter(log_writer_->file());
assert(file != nullptr);
return file->contents_;
}
void Encode(const std::string& msg) { void Encode(const std::string& msg) {
ASSERT_OK(log_writer_->AddRecord(Slice(msg))); ASSERT_OK(log_writer_->AddRecord(Slice(msg)));

4
db/plain_table_db_test.cc
View File

@ -49,9 +49,9 @@ TEST_F(PlainTableKeyDecoderTest, ReadNonMmap) {
Slice contents(tmp); Slice contents(tmp);
test::StringSource* string_source = test::StringSource* string_source =
new test::StringSource(contents, 0, false); new test::StringSource(contents, 0, false);
std::unique_ptr<FSRandomAccessFile> holder(string_source);
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
test::GetRandomAccessFileReader(string_source)); new RandomAccessFileReader(std::move(holder), "test"));
std::unique_ptr<PlainTableReaderFileInfo> file_info( std::unique_ptr<PlainTableReaderFileInfo> file_info(
new PlainTableReaderFileInfo(std::move(file_reader), EnvOptions(), new PlainTableReaderFileInfo(std::move(file_reader), EnvOptions(),
kLength)); kLength));

28
db/table_properties_collector_test.cc
View File

@ -13,7 +13,6 @@
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "db/dbformat.h" #include "db/dbformat.h"
#include "env/composite_env_wrapper.h"
#include "file/sequence_file_reader.h" #include "file/sequence_file_reader.h"
#include "file/writable_file_writer.h" #include "file/writable_file_writer.h"
#include "options/cf_options.h" #include "options/cf_options.h"
@ -49,10 +48,9 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
int_tbl_prop_collector_factories, int_tbl_prop_collector_factories,
std::unique_ptr<WritableFileWriter>* writable, std::unique_ptr<WritableFileWriter>* writable,
std::unique_ptr<TableBuilder>* builder) { std::unique_ptr<TableBuilder>* builder) {
std::unique_ptr<WritableFile> wf(new test::StringSink); std::unique_ptr<FSWritableFile> wf(new test::StringSink);
writable->reset( writable->reset(
new WritableFileWriter(NewLegacyWritableFileWrapper(std::move(wf)), new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions()));
"" /* don't care */, EnvOptions()));
int unknown_level = -1; int unknown_level = -1;
builder->reset(NewTableBuilder( builder->reset(NewTableBuilder(
ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories, ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories,
@ -286,12 +284,13 @@ void TestCustomizedTablePropertiesCollector(
writer->Flush(); writer->Flush();
// -- Step 2: Read properties // -- Step 2: Read properties
LegacyWritableFileWrapper* file = test::StringSink* fwf =
static_cast<LegacyWritableFileWrapper*>(writer->writable_file()); static_cast<test::StringSink*>(writer->writable_file());
test::StringSink* fwf = static_cast<test::StringSink*>(file->target()); std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(fwf->contents()));
std::unique_ptr<RandomAccessFileReader> fake_file_reader( std::unique_ptr<RandomAccessFileReader> fake_file_reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), "test"));
new test::StringSource(fwf->contents())));
TableProperties* props; TableProperties* props;
Status s = ReadTableProperties(fake_file_reader.get(), fwf->contents().size(), Status s = ReadTableProperties(fake_file_reader.get(), fwf->contents().size(),
magic_number, ioptions, &props, magic_number, ioptions, &props,
@ -427,12 +426,13 @@ void TestInternalKeyPropertiesCollector(
ASSERT_OK(builder->Finish()); ASSERT_OK(builder->Finish());
writable->Flush(); writable->Flush();
LegacyWritableFileWrapper* file = test::StringSink* fwf =
static_cast<LegacyWritableFileWrapper*>(writable->writable_file()); static_cast<test::StringSink*>(writable->writable_file());
test::StringSink* fwf = static_cast<test::StringSink*>(file->target()); std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(fwf->contents()));
std::unique_ptr<RandomAccessFileReader> reader( std::unique_ptr<RandomAccessFileReader> reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), "test"));
new test::StringSource(fwf->contents())));
TableProperties* props; TableProperties* props;
Status s = Status s =
ReadTableProperties(reader.get(), fwf->contents().size(), magic_number, ReadTableProperties(reader.get(), fwf->contents().size(), magic_number,

43
file/readahead_raf.cc
View File

@ -11,15 +11,17 @@
#include <algorithm> #include <algorithm>
#include <mutex> #include <mutex>
#include "file/read_write_util.h" #include "file/read_write_util.h"
#include "rocksdb/file_system.h"
#include "util/aligned_buffer.h" #include "util/aligned_buffer.h"
#include "util/rate_limiter.h" #include "util/rate_limiter.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace { namespace {
class ReadaheadRandomAccessFile : public RandomAccessFile { class ReadaheadRandomAccessFile : public FSRandomAccessFile {
public: public:
ReadaheadRandomAccessFile(std::unique_ptr<RandomAccessFile>&& file, ReadaheadRandomAccessFile(std::unique_ptr<FSRandomAccessFile>&& file,
size_t readahead_size) size_t readahead_size)
: file_(std::move(file)), : file_(std::move(file)),
alignment_(file_->GetRequiredBufferAlignment()), alignment_(file_->GetRequiredBufferAlignment()),
@ -35,11 +37,12 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
ReadaheadRandomAccessFile& operator=(const ReadaheadRandomAccessFile&) = ReadaheadRandomAccessFile& operator=(const ReadaheadRandomAccessFile&) =
delete; delete;
Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) const override { Slice* result, char* scratch,
IODebugContext* dbg) const override {
// Read-ahead only make sense if we have some slack left after reading // Read-ahead only make sense if we have some slack left after reading
if (n + alignment_ >= readahead_size_) { if (n + alignment_ >= readahead_size_) {
return file_->Read(offset, n, result, scratch); return file_->Read(offset, n, options, result, scratch, dbg);
} }
std::unique_lock<std::mutex> lk(lock_); std::unique_lock<std::mutex> lk(lock_);
@ -53,14 +56,14 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
(cached_len == n || buffer_.CurrentSize() < readahead_size_)) { (cached_len == n || buffer_.CurrentSize() < readahead_size_)) {
// We read exactly what we needed, or we hit end of file - return. // We read exactly what we needed, or we hit end of file - return.
*result = Slice(scratch, cached_len); *result = Slice(scratch, cached_len);
return Status::OK(); return IOStatus::OK();
} }
size_t advanced_offset = static_cast<size_t>(offset + cached_len); size_t advanced_offset = static_cast<size_t>(offset + cached_len);
// In the case of cache hit advanced_offset is already aligned, means that // In the case of cache hit advanced_offset is already aligned, means that
// chunk_offset equals to advanced_offset // chunk_offset equals to advanced_offset
size_t chunk_offset = TruncateToPageBoundary(alignment_, advanced_offset); size_t chunk_offset = TruncateToPageBoundary(alignment_, advanced_offset);
Status s = ReadIntoBuffer(chunk_offset, readahead_size_); IOStatus s = ReadIntoBuffer(chunk_offset, readahead_size_, options, dbg);
if (s.ok()) { if (s.ok()) {
// The data we need is now in cache, so we can safely read it // The data we need is now in cache, so we can safely read it
size_t remaining_len; size_t remaining_len;
@ -71,11 +74,12 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
return s; return s;
} }
Status Prefetch(uint64_t offset, size_t n) override { IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& options,
IODebugContext* dbg) override {
if (n < readahead_size_) { if (n < readahead_size_) {
// Don't allow smaller prefetches than the configured `readahead_size_`. // Don't allow smaller prefetches than the configured `readahead_size_`.
// `Read()` assumes a smaller prefetch buffer indicates EOF was reached. // `Read()` assumes a smaller prefetch buffer indicates EOF was reached.
return Status::OK(); return IOStatus::OK();
} }
std::unique_lock<std::mutex> lk(lock_); std::unique_lock<std::mutex> lk(lock_);
@ -83,10 +87,11 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
size_t offset_ = static_cast<size_t>(offset); size_t offset_ = static_cast<size_t>(offset);
size_t prefetch_offset = TruncateToPageBoundary(alignment_, offset_); size_t prefetch_offset = TruncateToPageBoundary(alignment_, offset_);
if (prefetch_offset == buffer_offset_) { if (prefetch_offset == buffer_offset_) {
return Status::OK(); return IOStatus::OK();
} }
return ReadIntoBuffer(prefetch_offset, return ReadIntoBuffer(prefetch_offset,
Roundup(offset_ + n, alignment_) - prefetch_offset); Roundup(offset_ + n, alignment_) - prefetch_offset,
options, dbg);
} }
size_t GetUniqueId(char* id, size_t max_size) const override { size_t GetUniqueId(char* id, size_t max_size) const override {
@ -95,7 +100,7 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
void Hint(AccessPattern pattern) override { file_->Hint(pattern); } void Hint(AccessPattern pattern) override { file_->Hint(pattern); }
Status InvalidateCache(size_t offset, size_t length) override { IOStatus InvalidateCache(size_t offset, size_t length) override {
std::unique_lock<std::mutex> lk(lock_); std::unique_lock<std::mutex> lk(lock_);
buffer_.Clear(); buffer_.Clear();
return file_->InvalidateCache(offset, length); return file_->InvalidateCache(offset, length);
@ -125,14 +130,16 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
// Reads into buffer_ the next n bytes from file_ starting at offset. // Reads into buffer_ the next n bytes from file_ starting at offset.
// Can actually read less if EOF was reached. // Can actually read less if EOF was reached.
// Returns the status of the read operastion on the file. // Returns the status of the read operastion on the file.
Status ReadIntoBuffer(uint64_t offset, size_t n) const { IOStatus ReadIntoBuffer(uint64_t offset, size_t n, const IOOptions& options,
IODebugContext* dbg) const {
if (n > buffer_.Capacity()) { if (n > buffer_.Capacity()) {
n = buffer_.Capacity(); n = buffer_.Capacity();
} }
assert(IsFileSectorAligned(offset, alignment_)); assert(IsFileSectorAligned(offset, alignment_));
assert(IsFileSectorAligned(n, alignment_)); assert(IsFileSectorAligned(n, alignment_));
Slice result; Slice result;
Status s = file_->Read(offset, n, &result, buffer_.BufferStart()); IOStatus s =
file_->Read(offset, n, options, &result, buffer_.BufferStart(), dbg);
if (s.ok()) { if (s.ok()) {
buffer_offset_ = offset; buffer_offset_ = offset;
buffer_.Size(result.size()); buffer_.Size(result.size());
@ -141,7 +148,7 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
return s; return s;
} }
const std::unique_ptr<RandomAccessFile> file_; const std::unique_ptr<FSRandomAccessFile> file_;
const size_t alignment_; const size_t alignment_;
const size_t readahead_size_; const size_t readahead_size_;
@ -153,9 +160,9 @@ class ReadaheadRandomAccessFile : public RandomAccessFile {
}; };
} // namespace } // namespace
std::unique_ptr<RandomAccessFile> NewReadaheadRandomAccessFile( std::unique_ptr<FSRandomAccessFile> NewReadaheadRandomAccessFile(
std::unique_ptr<RandomAccessFile>&& file, size_t readahead_size) { std::unique_ptr<FSRandomAccessFile>&& file, size_t readahead_size) {
std::unique_ptr<RandomAccessFile> result( std::unique_ptr<FSRandomAccessFile> result(
new ReadaheadRandomAccessFile(std::move(file), readahead_size)); new ReadaheadRandomAccessFile(std::move(file), readahead_size));
return result; return result;
} }

10
file/readahead_raf.h
View File

@ -8,10 +8,12 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once #pragma once
#include <atomic> #include <memory>
#include "rocksdb/env.h"
#include "rocksdb/rocksdb_namespace.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class FSRandomAccessFile;
// This file provides the following main abstractions: // This file provides the following main abstractions:
// SequentialFileReader : wrapper over Env::SequentialFile // SequentialFileReader : wrapper over Env::SequentialFile
// RandomAccessFileReader : wrapper over Env::RandomAccessFile // RandomAccessFileReader : wrapper over Env::RandomAccessFile
@ -22,6 +24,6 @@ namespace ROCKSDB_NAMESPACE {
// NewReadaheadRandomAccessFile provides a wrapper over RandomAccessFile to // NewReadaheadRandomAccessFile provides a wrapper over RandomAccessFile to
// always prefetch additional data with every read. This is mainly used in // always prefetch additional data with every read. This is mainly used in
// Compaction Table Readers. // Compaction Table Readers.
std::unique_ptr<RandomAccessFile> NewReadaheadRandomAccessFile( std::unique_ptr<FSRandomAccessFile> NewReadaheadRandomAccessFile(
std::unique_ptr<RandomAccessFile>&& file, size_t readahead_size); std::unique_ptr<FSRandomAccessFile>&& file, size_t readahead_size);
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

56
options/options_test.cc
View File

@ -2414,14 +2414,10 @@ TEST_F(OptionsOldApiTest, ColumnFamilyOptionsSerialization) {
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
class OptionsParserTest : public testing::Test { class OptionsParserTest : public testing::Test {
public: public:
OptionsParserTest() { OptionsParserTest() { fs_.reset(new test::StringFS(FileSystem::Default())); }
env_.reset(new test::StringEnv(Env::Default()));
fs_.reset(new LegacyFileSystemWrapper(env_.get()));
}
protected: protected:
std::unique_ptr<test::StringEnv> env_; std::shared_ptr<test::StringFS> fs_;
std::unique_ptr<LegacyFileSystemWrapper> fs_;
}; };
TEST_F(OptionsParserTest, Comment) { TEST_F(OptionsParserTest, Comment) {
@ -2450,7 +2446,7 @@ TEST_F(OptionsParserTest, Comment) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_OK( ASSERT_OK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2481,7 +2477,7 @@ TEST_F(OptionsParserTest, ExtraSpace) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_OK( ASSERT_OK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2499,7 +2495,7 @@ TEST_F(OptionsParserTest, MissingDBOptions) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_NOK( ASSERT_NOK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2529,7 +2525,7 @@ TEST_F(OptionsParserTest, DoubleDBOptions) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_NOK( ASSERT_NOK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2557,7 +2553,7 @@ TEST_F(OptionsParserTest, NoDefaultCFOptions) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_NOK( ASSERT_NOK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2587,7 +2583,7 @@ TEST_F(OptionsParserTest, DefaultCFOptionsMustBeTheFirst) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_NOK( ASSERT_NOK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2616,7 +2612,7 @@ TEST_F(OptionsParserTest, DuplicateCFOptions) {
"[CFOptions \"something_else\"]\n"; "[CFOptions \"something_else\"]\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_NOK( ASSERT_NOK(
parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */)); parser.Parse(kTestFileName, fs_.get(), false, 4096 /* readahead_size */));
@ -2684,12 +2680,12 @@ TEST_F(OptionsParserTest, IgnoreUnknownOptions) {
" # if a section is blank, we will use the default\n"; " # if a section is blank, we will use the default\n";
const std::string kTestFileName = "test-rocksdb-options.ini"; const std::string kTestFileName = "test-rocksdb-options.ini";
auto s = env_->FileExists(kTestFileName); auto s = fs_->FileExists(kTestFileName, IOOptions(), nullptr);
ASSERT_TRUE(s.ok() || s.IsNotFound()); ASSERT_TRUE(s.ok() || s.IsNotFound());
if (s.ok()) { if (s.ok()) {
ASSERT_OK(env_->DeleteFile(kTestFileName)); ASSERT_OK(fs_->DeleteFile(kTestFileName, IOOptions(), nullptr));
} }
ASSERT_OK(env_->WriteToNewFile(kTestFileName, options_file_content)); ASSERT_OK(fs_->WriteToNewFile(kTestFileName, options_file_content));
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false, ASSERT_NOK(parser.Parse(kTestFileName, fs_.get(), false,
4096 /* readahead_size */)); 4096 /* readahead_size */));
@ -2737,7 +2733,7 @@ TEST_F(OptionsParserTest, ParseVersion) {
snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str()); snprintf(buffer, kLength - 1, file_template.c_str(), iv.c_str());
parser.Reset(); parser.Reset();
ASSERT_OK(env_->WriteToNewFile(iv, buffer)); ASSERT_OK(fs_->WriteToNewFile(iv, buffer));
ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */)); ASSERT_NOK(parser.Parse(iv, fs_.get(), false, 0 /* readahead_size */));
} }
@ -2746,7 +2742,7 @@ TEST_F(OptionsParserTest, ParseVersion) {
for (auto vv : valid_versions) { for (auto vv : valid_versions) {
snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str()); snprintf(buffer, kLength - 1, file_template.c_str(), vv.c_str());
parser.Reset(); parser.Reset();
ASSERT_OK(env_->WriteToNewFile(vv, buffer)); ASSERT_OK(fs_->WriteToNewFile(vv, buffer));
ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */)); ASSERT_OK(parser.Parse(vv, fs_.get(), false, 0 /* readahead_size */));
} }
} }
@ -2855,37 +2851,37 @@ TEST_F(OptionsParserTest, Readahead) {
kOptionsFileName, fs_.get())); kOptionsFileName, fs_.get()));
uint64_t file_size = 0; uint64_t file_size = 0;
ASSERT_OK(env_->GetFileSize(kOptionsFileName, &file_size)); ASSERT_OK(
fs_->GetFileSize(kOptionsFileName, IOOptions(), &file_size, nullptr));
assert(file_size > 0); assert(file_size > 0);
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
env_->num_seq_file_read_ = 0; fs_->num_seq_file_read_ = 0;
size_t readahead_size = 128 * 1024; size_t readahead_size = 128 * 1024;
ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size)); ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
ASSERT_EQ(env_->num_seq_file_read_.load(), ASSERT_EQ(fs_->num_seq_file_read_.load(),
(file_size - 1) / readahead_size + 1); (file_size - 1) / readahead_size + 1);
env_->num_seq_file_read_.store(0); fs_->num_seq_file_read_.store(0);
readahead_size = 1024 * 1024; readahead_size = 1024 * 1024;
ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size)); ASSERT_OK(parser.Parse(kOptionsFileName, fs_.get(), false, readahead_size));
ASSERT_EQ(env_->num_seq_file_read_.load(), ASSERT_EQ(fs_->num_seq_file_read_.load(),
(file_size - 1) / readahead_size + 1); (file_size - 1) / readahead_size + 1);
// Tiny readahead. 8 KB is read each time. // Tiny readahead. 8 KB is read each time.
env_->num_seq_file_read_.store(0); fs_->num_seq_file_read_.store(0);
ASSERT_OK( ASSERT_OK(
parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */)); parser.Parse(kOptionsFileName, fs_.get(), false, 1 /* readahead_size */));
ASSERT_GE(env_->num_seq_file_read_.load(), file_size / (8 * 1024)); ASSERT_GE(fs_->num_seq_file_read_.load(), file_size / (8 * 1024));
ASSERT_LT(env_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2); ASSERT_LT(fs_->num_seq_file_read_.load(), file_size / (8 * 1024) * 2);
// Disable readahead means 512KB readahead. // Disable readahead means 512KB readahead.
env_->num_seq_file_read_.store(0); fs_->num_seq_file_read_.store(0);
ASSERT_OK( ASSERT_OK(
parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */)); parser.Parse(kOptionsFileName, fs_.get(), false, 0 /* readahead_size */));
ASSERT_GE(env_->num_seq_file_read_.load(), ASSERT_GE(fs_->num_seq_file_read_.load(), (file_size - 1) / (512 * 1024) + 1);
(file_size - 1) / (512 * 1024) + 1);
} }
TEST_F(OptionsParserTest, DumpAndParse) { TEST_F(OptionsParserTest, DumpAndParse) {
@ -3083,7 +3079,7 @@ class OptionsSanityCheckTest : public OptionsParserTest {
} }
Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) { Status PersistCFOptions(const ColumnFamilyOptions& cf_opts) {
Status s = env_->DeleteFile(kOptionsFileName); Status s = fs_->DeleteFile(kOptionsFileName, IOOptions(), nullptr);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }

19
table/block_based/data_block_hash_index_test.cc
View File

@ -546,8 +546,10 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
EnvOptions soptions; EnvOptions soptions;
soptions.use_mmap_reads = ioptions.allow_mmap_reads; soptions.use_mmap_reads = ioptions.allow_mmap_reads;
test::StringSink* sink = new test::StringSink();
std::unique_ptr<FSWritableFile> f(sink);
file_writer.reset( file_writer.reset(
test::GetWritableFileWriter(new test::StringSink(), "" /* don't care */)); new WritableFileWriter(std::move(f), "" /* don't care */, FileOptions()));
std::unique_ptr<TableBuilder> builder; std::unique_ptr<TableBuilder> builder;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories; int_tbl_prop_collector_factories;
@ -569,23 +571,20 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2,
file_writer->Flush(); file_writer->Flush();
EXPECT_TRUE(s.ok()) << s.ToString(); EXPECT_TRUE(s.ok()) << s.ToString();
EXPECT_EQ( EXPECT_EQ(sink->contents().size(), builder->FileSize());
test::GetStringSinkFromLegacyWriter(file_writer.get())->contents().size(),
builder->FileSize());
// Open the table // Open the table
file_reader.reset(test::GetRandomAccessFileReader(new test::StringSource( test::StringSource* source = new test::StringSource(
test::GetStringSinkFromLegacyWriter(file_writer.get())->contents(), sink->contents(), 0 /*uniq_id*/, ioptions.allow_mmap_reads);
0 /*uniq_id*/, ioptions.allow_mmap_reads))); std::unique_ptr<FSRandomAccessFile> file(source);
file_reader.reset(new RandomAccessFileReader(std::move(file), "test"));
const bool kSkipFilters = true; const bool kSkipFilters = true;
const bool kImmortal = true; const bool kImmortal = true;
ASSERT_OK(ioptions.table_factory->NewTableReader( ASSERT_OK(ioptions.table_factory->NewTableReader(
TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions, TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions,
internal_comparator, !kSkipFilters, !kImmortal, internal_comparator, !kSkipFilters, !kImmortal,
level_), level_),
std::move(file_reader), std::move(file_reader), sink->contents().size(), &table_reader));
test::GetStringSinkFromLegacyWriter(file_writer.get())->contents().size(),
&table_reader));
// Search using Get() // Search using Get()
ReadOptions ro; ReadOptions ro;

106
table/table_test.cc
View File

@ -345,8 +345,9 @@ class TableConstructor : public Constructor {
const stl_wrappers::KVMap& kv_map) override { const stl_wrappers::KVMap& kv_map) override {
Reset(); Reset();
soptions.use_mmap_reads = ioptions.allow_mmap_reads; soptions.use_mmap_reads = ioptions.allow_mmap_reads;
file_writer_.reset(test::GetWritableFileWriter(new test::StringSink(), std::unique_ptr<FSWritableFile> sink(new test::StringSink());
"" /* don't care */)); file_writer_.reset(new WritableFileWriter(
std::move(sink), "" /* don't care */, FileOptions()));
std::unique_ptr<TableBuilder> builder; std::unique_ptr<TableBuilder> builder;
std::vector<std::unique_ptr<IntTblPropCollectorFactory>> std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
int_tbl_prop_collector_factories; int_tbl_prop_collector_factories;
@ -387,8 +388,10 @@ class TableConstructor : public Constructor {
// Open the table // Open the table
uniq_id_ = cur_uniq_id_++; uniq_id_ = cur_uniq_id_++;
file_reader_.reset(test::GetRandomAccessFileReader(new test::StringSource( std::unique_ptr<FSRandomAccessFile> source(new test::StringSource(
TEST_GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads))); TEST_GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads));
file_reader_.reset(new RandomAccessFileReader(std::move(source), "test"));
const bool kSkipFilters = true; const bool kSkipFilters = true;
const bool kImmortal = true; const bool kImmortal = true;
return ioptions.table_factory->NewTableReader( return ioptions.table_factory->NewTableReader(
@ -425,8 +428,10 @@ class TableConstructor : public Constructor {
virtual Status Reopen(const ImmutableCFOptions& ioptions, virtual Status Reopen(const ImmutableCFOptions& ioptions,
const MutableCFOptions& moptions) { const MutableCFOptions& moptions) {
file_reader_.reset(test::GetRandomAccessFileReader(new test::StringSource( std::unique_ptr<FSRandomAccessFile> source(new test::StringSource(
TEST_GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads))); TEST_GetSink()->contents(), uniq_id_, ioptions.allow_mmap_reads));
file_reader_.reset(new RandomAccessFileReader(std::move(source), "test"));
return ioptions.table_factory->NewTableReader( return ioptions.table_factory->NewTableReader(
TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions, TableReaderOptions(ioptions, moptions.prefix_extractor.get(), soptions,
*last_internal_key_), *last_internal_key_),
@ -445,8 +450,7 @@ class TableConstructor : public Constructor {
bool ConvertToInternalKey() { return convert_to_internal_key_; } bool ConvertToInternalKey() { return convert_to_internal_key_; }
test::StringSink* TEST_GetSink() { test::StringSink* TEST_GetSink() {
return ROCKSDB_NAMESPACE::test::GetStringSinkFromLegacyWriter( return static_cast<test::StringSink*>(file_writer_->writable_file());
file_writer_.get());
} }
BlockCacheTracer block_cache_tracer_; BlockCacheTracer block_cache_tracer_;
@ -1230,7 +1234,9 @@ class FileChecksumTestHelper {
void CreateWriteableFile() { void CreateWriteableFile() {
sink_ = new test::StringSink(); sink_ = new test::StringSink();
file_writer_.reset(test::GetWritableFileWriter(sink_, "" /* don't care */)); std::unique_ptr<FSWritableFile> holder(sink_);
file_writer_.reset(new WritableFileWriter(
std::move(holder), "" /* don't care */, FileOptions()));
} }
void SetFileChecksumGenerator(FileChecksumGenerator* checksum_generator) { void SetFileChecksumGenerator(FileChecksumGenerator* checksum_generator) {
@ -1291,10 +1297,11 @@ class FileChecksumTestHelper {
assert(file_checksum_generator != nullptr); assert(file_checksum_generator != nullptr);
cur_uniq_id_ = checksum_uniq_id_++; cur_uniq_id_ = checksum_uniq_id_++;
test::StringSink* ss_rw = test::StringSink* ss_rw =
ROCKSDB_NAMESPACE::test::GetStringSinkFromLegacyWriter( static_cast<test::StringSink*>(file_writer_->writable_file());
file_writer_.get()); std::unique_ptr<FSRandomAccessFile> source(
file_reader_.reset(test::GetRandomAccessFileReader( new test::StringSource(ss_rw->contents()));
new test::StringSource(ss_rw->contents()))); file_reader_.reset(new RandomAccessFileReader(std::move(source), "test"));
std::unique_ptr<char[]> scratch(new char[2048]); std::unique_ptr<char[]> scratch(new char[2048]);
Slice result; Slice result;
uint64_t offset = 0; uint64_t offset = 0;
@ -3392,9 +3399,9 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
plain_table_options.hash_table_ratio = 0; plain_table_options.hash_table_ratio = 0;
PlainTableFactory factory(plain_table_options); PlainTableFactory factory(plain_table_options);
test::StringSink sink; std::unique_ptr<FSWritableFile> sink(new test::StringSink());
std::unique_ptr<WritableFileWriter> file_writer( std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
test::GetWritableFileWriter(new test::StringSink(), "" /* don't care */)); std::move(sink), "" /* don't care */, FileOptions()));
Options options; Options options;
const ImmutableCFOptions ioptions(options); const ImmutableCFOptions ioptions(options);
const MutableCFOptions moptions(options); const MutableCFOptions moptions(options);
@ -3421,10 +3428,11 @@ TEST_F(PlainTableTest, BasicPlainTableProperties) {
ASSERT_OK(file_writer->Flush()); ASSERT_OK(file_writer->Flush());
test::StringSink* ss = test::StringSink* ss =
ROCKSDB_NAMESPACE::test::GetStringSinkFromLegacyWriter(file_writer.get()); static_cast<test::StringSink*>(file_writer->writable_file());
std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(ss->contents(), 72242, true));
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), "test"));
new test::StringSource(ss->contents(), 72242, true)));
TableProperties* props = nullptr; TableProperties* props = nullptr;
auto s = ReadTableProperties(file_reader.get(), ss->contents().size(), auto s = ReadTableProperties(file_reader.get(), ss->contents().size(),
@ -4052,8 +4060,9 @@ TEST_F(PrefixTest, PrefixAndWholeKeyTest) {
TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) { TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) {
BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
test::StringSink* sink = new test::StringSink(); test::StringSink* sink = new test::StringSink();
std::unique_ptr<WritableFileWriter> file_writer( std::unique_ptr<FSWritableFile> holder(sink);
test::GetWritableFileWriter(sink, "" /* don't care */)); std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(holder), "" /* don't care */, FileOptions()));
Options options; Options options;
options.table_factory.reset(NewBlockBasedTableFactory(bbto)); options.table_factory.reset(NewBlockBasedTableFactory(bbto));
const ImmutableCFOptions ioptions(options); const ImmutableCFOptions ioptions(options);
@ -4090,9 +4099,10 @@ TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) {
// Helper function to get version, global_seqno, global_seqno_offset // Helper function to get version, global_seqno, global_seqno_offset
std::function<void()> GetVersionAndGlobalSeqno = [&]() { std::function<void()> GetVersionAndGlobalSeqno = [&]() {
std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(ss_rw.contents(), 73342, true));
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), ""));
new test::StringSource(ss_rw.contents(), 73342, true)));
TableProperties* props = nullptr; TableProperties* props = nullptr;
ASSERT_OK(ReadTableProperties(file_reader.get(), ss_rw.contents().size(), ASSERT_OK(ReadTableProperties(file_reader.get(), ss_rw.contents().size(),
@ -4115,16 +4125,18 @@ TEST_P(BlockBasedTableTest, DISABLED_TableWithGlobalSeqno) {
std::string new_global_seqno; std::string new_global_seqno;
PutFixed64(&new_global_seqno, val); PutFixed64(&new_global_seqno, val);
ASSERT_OK(ss_rw.Write(global_seqno_offset, new_global_seqno)); ASSERT_OK(ss_rw.Write(global_seqno_offset, new_global_seqno, IOOptions(),
nullptr));
}; };
// Helper function to get the contents of the table InternalIterator // Helper function to get the contents of the table InternalIterator
std::unique_ptr<TableReader> table_reader; std::unique_ptr<TableReader> table_reader;
const ReadOptions read_options; const ReadOptions read_options;
std::function<InternalIterator*()> GetTableInternalIter = [&]() { std::function<InternalIterator*()> GetTableInternalIter = [&]() {
std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(ss_rw.contents(), 73342, true));
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), ""));
new test::StringSource(ss_rw.contents(), 73342, true)));
options.table_factory->NewTableReader( options.table_factory->NewTableReader(
TableReaderOptions(ioptions, moptions.prefix_extractor.get(), TableReaderOptions(ioptions, moptions.prefix_extractor.get(),
@ -4236,8 +4248,9 @@ TEST_P(BlockBasedTableTest, BlockAlignTest) {
BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
bbto.block_align = true; bbto.block_align = true;
test::StringSink* sink = new test::StringSink(); test::StringSink* sink = new test::StringSink();
std::unique_ptr<WritableFileWriter> file_writer( std::unique_ptr<FSWritableFile> holder(sink);
test::GetWritableFileWriter(sink, "" /* don't care */)); std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(holder), "" /* don't care */, FileOptions()));
Options options; Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
options.table_factory.reset(NewBlockBasedTableFactory(bbto)); options.table_factory.reset(NewBlockBasedTableFactory(bbto));
@ -4267,17 +4280,16 @@ TEST_P(BlockBasedTableTest, BlockAlignTest) {
ASSERT_OK(builder->Finish()); ASSERT_OK(builder->Finish());
ASSERT_OK(file_writer->Flush()); ASSERT_OK(file_writer->Flush());
test::RandomRWStringSink ss_rw(sink); std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(sink->contents(), 73342, false));
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), "test"));
new test::StringSource(ss_rw.contents(), 73342, true)));
// Helper function to get version, global_seqno, global_seqno_offset // Helper function to get version, global_seqno, global_seqno_offset
std::function<void()> VerifyBlockAlignment = [&]() { std::function<void()> VerifyBlockAlignment = [&]() {
TableProperties* props = nullptr; TableProperties* props = nullptr;
ASSERT_OK(ReadTableProperties(file_reader.get(), ss_rw.contents().size(), ASSERT_OK(ReadTableProperties(file_reader.get(), sink->contents().size(),
kBlockBasedTableMagicNumber, ioptions, kBlockBasedTableMagicNumber, ioptions, &props,
&props, true /* compression_type_missing */)); true /* compression_type_missing */));
uint64_t data_block_size = props->data_size / props->num_data_blocks; uint64_t data_block_size = props->data_size / props->num_data_blocks;
ASSERT_EQ(data_block_size, 4096); ASSERT_EQ(data_block_size, 4096);
@ -4301,7 +4313,7 @@ TEST_P(BlockBasedTableTest, BlockAlignTest) {
TableReaderOptions(ioptions2, moptions2.prefix_extractor.get(), TableReaderOptions(ioptions2, moptions2.prefix_extractor.get(),
EnvOptions(), EnvOptions(),
GetPlainInternalComparator(options2.comparator)), GetPlainInternalComparator(options2.comparator)),
std::move(file_reader), ss_rw.contents().size(), &table_reader)); std::move(file_reader), sink->contents().size(), &table_reader));
ReadOptions read_options; ReadOptions read_options;
std::unique_ptr<InternalIterator> db_iter(table_reader->NewIterator( std::unique_ptr<InternalIterator> db_iter(table_reader->NewIterator(
@ -4328,8 +4340,9 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
BlockBasedTableOptions bbto = GetBlockBasedTableOptions(); BlockBasedTableOptions bbto = GetBlockBasedTableOptions();
bbto.block_align = true; bbto.block_align = true;
test::StringSink* sink = new test::StringSink(); test::StringSink* sink = new test::StringSink();
std::unique_ptr<WritableFileWriter> file_writer( std::unique_ptr<FSWritableFile> holder(sink);
test::GetWritableFileWriter(sink, "" /* don't care */)); std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
std::move(holder), "" /* don't care */, FileOptions()));
Options options; Options options;
options.compression = kNoCompression; options.compression = kNoCompression;
@ -4362,14 +4375,14 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
ASSERT_OK(builder->Finish()); ASSERT_OK(builder->Finish());
ASSERT_OK(file_writer->Flush()); ASSERT_OK(file_writer->Flush());
test::RandomRWStringSink ss_rw(sink); std::unique_ptr<FSRandomAccessFile> source(
new test::StringSource(sink->contents(), 73342, true));
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
test::GetRandomAccessFileReader( new RandomAccessFileReader(std::move(source), "test"));
new test::StringSource(ss_rw.contents(), 73342, true)));
{ {
RandomAccessFileReader* file = file_reader.get(); RandomAccessFileReader* file = file_reader.get();
uint64_t file_size = ss_rw.contents().size(); uint64_t file_size = sink->contents().size();
Footer footer; Footer footer;
IOOptions opts; IOOptions opts;
@ -4452,10 +4465,11 @@ TEST_P(BlockBasedTableTest, PropertiesMetaBlockLast) {
// get file reader // get file reader
test::StringSink* table_sink = c.TEST_GetSink(); test::StringSink* table_sink = c.TEST_GetSink();
std::unique_ptr<RandomAccessFileReader> table_reader{ std::unique_ptr<FSRandomAccessFile> source(new test::StringSource(
test::GetRandomAccessFileReader( table_sink->contents(), 0 /* unique_id */, false /* allow_mmap_reads */));
new test::StringSource(table_sink->contents(), 0 /* unique_id */,
false /* allow_mmap_reads */))}; std::unique_ptr<RandomAccessFileReader> table_reader(
new RandomAccessFileReader(std::move(source), "test"));
size_t table_size = table_sink->contents().size(); size_t table_size = table_sink->contents().size();
// read footer // read footer

19
test_util/testutil.cc
View File

@ -171,25 +171,6 @@ const Comparator* ComparatorWithU64Ts() {
return &comp_with_u64_ts; return &comp_with_u64_ts;
} }
WritableFileWriter* GetWritableFileWriter(WritableFile* wf,
const std::string& fname) {
std::unique_ptr<WritableFile> file(wf);
return new WritableFileWriter(NewLegacyWritableFileWrapper(std::move(file)),
fname, EnvOptions());
}
RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf) {
std::unique_ptr<RandomAccessFile> file(raf);
return new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(file),
"[test RandomAccessFileReader]");
}
SequentialFileReader* GetSequentialFileReader(SequentialFile* se,
const std::string& fname) {
std::unique_ptr<SequentialFile> file(se);
return new SequentialFileReader(NewLegacySequentialFileWrapper(file), fname);
}
void CorruptKeyType(InternalKey* ikey) { void CorruptKeyType(InternalKey* ikey) {
std::string keystr = ikey->Encode().ToString(); std::string keystr = ikey->Encode().ToString();
keystr[keystr.size() - 8] = kTypeLogData; keystr[keystr.size() - 8] = kTypeLogData;

464
test_util/testutil.h
View File

@ -178,23 +178,16 @@ class VectorIterator : public InternalIterator {
std::vector<std::string> values_; std::vector<std::string> values_;
size_t current_; size_t current_;
}; };
extern WritableFileWriter* GetWritableFileWriter(WritableFile* wf,
const std::string& fname);
extern RandomAccessFileReader* GetRandomAccessFileReader(RandomAccessFile* raf); class StringSink : public FSWritableFile {
extern SequentialFileReader* GetSequentialFileReader(SequentialFile* se,
const std::string& fname);
class StringSink: public WritableFile {
public: public:
std::string contents_; std::string contents_;
explicit StringSink(Slice* reader_contents = nullptr) : explicit StringSink(Slice* reader_contents = nullptr)
WritableFile(), : FSWritableFile(),
contents_(""), contents_(""),
reader_contents_(reader_contents), reader_contents_(reader_contents),
last_flush_(0) { last_flush_(0) {
if (reader_contents_ != nullptr) { if (reader_contents_ != nullptr) {
*reader_contents_ = Slice(contents_.data(), 0); *reader_contents_ = Slice(contents_.data(), 0);
} }
@ -202,12 +195,15 @@ class StringSink: public WritableFile {
const std::string& contents() const { return contents_; } const std::string& contents() const { return contents_; }
virtual Status Truncate(uint64_t size) override { IOStatus Truncate(uint64_t size, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
contents_.resize(static_cast<size_t>(size)); contents_.resize(static_cast<size_t>(size));
return Status::OK(); return IOStatus::OK();
} }
virtual Status Close() override { return Status::OK(); } IOStatus Close(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
virtual Status Flush() override { return IOStatus::OK();
}
IOStatus Flush(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
if (reader_contents_ != nullptr) { if (reader_contents_ != nullptr) {
assert(reader_contents_->size() <= last_flush_); assert(reader_contents_->size() <= last_flush_);
size_t offset = last_flush_ - reader_contents_->size(); size_t offset = last_flush_ - reader_contents_->size();
@ -217,12 +213,17 @@ class StringSink: public WritableFile {
last_flush_ = contents_.size(); last_flush_ = contents_.size();
} }
return Status::OK(); return IOStatus::OK();
} }
virtual Status Sync() override { return Status::OK(); } IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
virtual Status Append(const Slice& slice) override { return IOStatus::OK();
}
using FSWritableFile::Append;
IOStatus Append(const Slice& slice, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
contents_.append(slice.data(), slice.size()); contents_.append(slice.data(), slice.size());
return Status::OK(); return IOStatus::OK();
} }
void Drop(size_t bytes) { void Drop(size_t bytes) {
if (reader_contents_ != nullptr) { if (reader_contents_ != nullptr) {
@ -239,36 +240,44 @@ class StringSink: public WritableFile {
}; };
// A wrapper around a StringSink to give it a RandomRWFile interface // A wrapper around a StringSink to give it a RandomRWFile interface
class RandomRWStringSink : public RandomRWFile { class RandomRWStringSink : public FSRandomRWFile {
public: public:
explicit RandomRWStringSink(StringSink* ss) : ss_(ss) {} explicit RandomRWStringSink(StringSink* ss) : ss_(ss) {}
Status Write(uint64_t offset, const Slice& data) override { IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
if (offset + data.size() > ss_->contents_.size()) { if (offset + data.size() > ss_->contents_.size()) {
ss_->contents_.resize(static_cast<size_t>(offset) + data.size(), '\0'); ss_->contents_.resize(static_cast<size_t>(offset) + data.size(), '\0');
} }
char* pos = const_cast<char*>(ss_->contents_.data() + offset); char* pos = const_cast<char*>(ss_->contents_.data() + offset);
memcpy(pos, data.data(), data.size()); memcpy(pos, data.data(), data.size());
return Status::OK(); return IOStatus::OK();
} }
Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*opts*/,
char* /*scratch*/) const override { Slice* result, char* /*scratch*/,
IODebugContext* /*dbg*/) const override {
*result = Slice(nullptr, 0); *result = Slice(nullptr, 0);
if (offset < ss_->contents_.size()) { if (offset < ss_->contents_.size()) {
size_t str_res_sz = size_t str_res_sz =
std::min(static_cast<size_t>(ss_->contents_.size() - offset), n); std::min(static_cast<size_t>(ss_->contents_.size() - offset), n);
*result = Slice(ss_->contents_.data() + offset, str_res_sz); *result = Slice(ss_->contents_.data() + offset, str_res_sz);
} }
return Status::OK(); return IOStatus::OK();
} }
Status Flush() override { return Status::OK(); } IOStatus Flush(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
return IOStatus::OK();
}
Status Sync() override { return Status::OK(); } IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
return IOStatus::OK();
}
Status Close() override { return Status::OK(); } IOStatus Close(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
return IOStatus::OK();
}
const std::string& contents() const { return ss_->contents(); } const std::string& contents() const { return ss_->contents(); }
@ -279,34 +288,42 @@ class RandomRWStringSink : public RandomRWFile {
// Like StringSink, this writes into a string. Unlink StringSink, it // Like StringSink, this writes into a string. Unlink StringSink, it
// has some initial content and overwrites it, just like a recycled // has some initial content and overwrites it, just like a recycled
// log file. // log file.
class OverwritingStringSink : public WritableFile { class OverwritingStringSink : public FSWritableFile {
public: public:
explicit OverwritingStringSink(Slice* reader_contents) explicit OverwritingStringSink(Slice* reader_contents)
: WritableFile(), : FSWritableFile(),
contents_(""), contents_(""),
reader_contents_(reader_contents), reader_contents_(reader_contents),
last_flush_(0) {} last_flush_(0) {}
const std::string& contents() const { return contents_; } const std::string& contents() const { return contents_; }
virtual Status Truncate(uint64_t size) override { IOStatus Truncate(uint64_t size, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
contents_.resize(static_cast<size_t>(size)); contents_.resize(static_cast<size_t>(size));
return Status::OK(); return IOStatus::OK();
} }
virtual Status Close() override { return Status::OK(); } IOStatus Close(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
virtual Status Flush() override { return IOStatus::OK();
}
IOStatus Flush(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
if (last_flush_ < contents_.size()) { if (last_flush_ < contents_.size()) {
assert(reader_contents_->size() >= contents_.size()); assert(reader_contents_->size() >= contents_.size());
memcpy((char*)reader_contents_->data() + last_flush_, memcpy((char*)reader_contents_->data() + last_flush_,
contents_.data() + last_flush_, contents_.size() - last_flush_); contents_.data() + last_flush_, contents_.size() - last_flush_);
last_flush_ = contents_.size(); last_flush_ = contents_.size();
} }
return Status::OK(); return IOStatus::OK();
} }
virtual Status Sync() override { return Status::OK(); } IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
virtual Status Append(const Slice& slice) override { return IOStatus::OK();
}
using FSWritableFile::Append;
IOStatus Append(const Slice& slice, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
contents_.append(slice.data(), slice.size()); contents_.append(slice.data(), slice.size());
return Status::OK(); return IOStatus::OK();
} }
void Drop(size_t bytes) { void Drop(size_t bytes) {
contents_.resize(contents_.size() - bytes); contents_.resize(contents_.size() - bytes);
@ -319,7 +336,7 @@ class OverwritingStringSink : public WritableFile {
size_t last_flush_; size_t last_flush_;
}; };
class StringSource: public RandomAccessFile { class StringSource : public FSRandomAccessFile {
public: public:
explicit StringSource(const Slice& contents, uint64_t uniq_id = 0, explicit StringSource(const Slice& contents, uint64_t uniq_id = 0,
bool mmap = false) bool mmap = false)
@ -332,11 +349,23 @@ class StringSource: public RandomAccessFile {
uint64_t Size() const { return contents_.size(); } uint64_t Size() const { return contents_.size(); }
virtual Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Prefetch(uint64_t /*offset*/, size_t /*n*/,
char* scratch) const override { const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
// If we are using mmap_, it is equivalent to performing a prefetch
if (mmap_) {
return IOStatus::OK();
} else {
return IOStatus::NotSupported("Prefetch not supported");
}
}
IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*opts*/,
Slice* result, char* scratch,
IODebugContext* /*dbg*/) const override {
total_reads_++; total_reads_++;
if (offset > contents_.size()) { if (offset > contents_.size()) {
return Status::InvalidArgument("invalid Read offset"); return IOStatus::InvalidArgument("invalid Read offset");
} }
if (offset + n > contents_.size()) { if (offset + n > contents_.size()) {
n = contents_.size() - static_cast<size_t>(offset); n = contents_.size() - static_cast<size_t>(offset);
@ -347,10 +376,10 @@ class StringSource: public RandomAccessFile {
} else { } else {
*result = Slice(&contents_[static_cast<size_t>(offset)], n); *result = Slice(&contents_[static_cast<size_t>(offset)], n);
} }
return Status::OK(); return IOStatus::OK();
} }
virtual size_t GetUniqueId(char* id, size_t max_size) const override { size_t GetUniqueId(char* id, size_t max_size) const override {
if (max_size < 20) { if (max_size < 20) {
return 0; return 0;
} }
@ -372,13 +401,6 @@ class StringSource: public RandomAccessFile {
mutable int total_reads_; mutable int total_reads_;
}; };
inline StringSink* GetStringSinkFromLegacyWriter(
const WritableFileWriter* writer) {
LegacyWritableFileWrapper* file =
static_cast<LegacyWritableFileWrapper*>(writer->writable_file());
return static_cast<StringSink*>(file->target());
}
class NullLogger : public Logger { class NullLogger : public Logger {
public: public:
using Logger::Logv; using Logger::Logv;
@ -525,176 +547,220 @@ inline std::string EncodeInt(uint64_t x) {
return result; return result;
} }
class SeqStringSource : public SequentialFile { class SeqStringSource : public FSSequentialFile {
public: public:
SeqStringSource(const std::string& data, std::atomic<int>* read_count) SeqStringSource(const std::string& data, std::atomic<int>* read_count)
: data_(data), offset_(0), read_count_(read_count) {} : data_(data), offset_(0), read_count_(read_count) {}
~SeqStringSource() override {} ~SeqStringSource() override {}
Status Read(size_t n, Slice* result, char* scratch) override { IOStatus Read(size_t n, const IOOptions& /*opts*/, Slice* result,
std::string output; char* scratch, IODebugContext* /*dbg*/) override {
if (offset_ < data_.size()) { std::string output;
n = std::min(data_.size() - offset_, n); if (offset_ < data_.size()) {
memcpy(scratch, data_.data() + offset_, n); n = std::min(data_.size() - offset_, n);
offset_ += n; memcpy(scratch, data_.data() + offset_, n);
*result = Slice(scratch, n); offset_ += n;
} else { *result = Slice(scratch, n);
return Status::InvalidArgument( } else {
"Attemp to read when it already reached eof."); return IOStatus::InvalidArgument(
} "Attempt to read when it already reached eof.");
(*read_count_)++;
return Status::OK();
} }
Status Skip(uint64_t n) override { (*read_count_)++;
if (offset_ >= data_.size()) { return IOStatus::OK();
return Status::InvalidArgument( }
"Attemp to read when it already reached eof.");
} IOStatus Skip(uint64_t n) override {
// TODO(yhchiang): Currently doesn't handle the overflow case. if (offset_ >= data_.size()) {
offset_ += static_cast<size_t>(n); return IOStatus::InvalidArgument(
return Status::OK(); "Attempt to read when it already reached eof.");
}
// TODO(yhchiang): Currently doesn't handle the overflow case.
offset_ += static_cast<size_t>(n);
return IOStatus::OK();
}
private:
std::string data_;
size_t offset_;
std::atomic<int>* read_count_;
};
class StringFS : public FileSystemWrapper {
public:
class StringSink : public FSWritableFile {
public:
explicit StringSink(std::string* contents)
: FSWritableFile(), contents_(contents) {}
IOStatus Truncate(uint64_t size, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
contents_->resize(static_cast<size_t>(size));
return IOStatus::OK();
}
IOStatus Close(const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
return IOStatus::OK();
}
IOStatus Flush(const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
return IOStatus::OK();
}
IOStatus Sync(const IOOptions& /*opts*/, IODebugContext* /*dbg*/) override {
return IOStatus::OK();
}
using FSWritableFile::Append;
IOStatus Append(const Slice& slice, const IOOptions& /*opts*/,
IODebugContext* /*dbg*/) override {
contents_->append(slice.data(), slice.size());
return IOStatus::OK();
} }
private: private:
std::string data_; std::string* contents_;
size_t offset_;
std::atomic<int>* read_count_;
}; };
class StringEnv : public EnvWrapper { explicit StringFS(const std::shared_ptr<FileSystem>& t)
public: : FileSystemWrapper(t) {}
class StringSink : public WritableFile { ~StringFS() override {}
public:
explicit StringSink(std::string* contents)
: WritableFile(), contents_(contents) {}
virtual Status Truncate(uint64_t size) override {
contents_->resize(static_cast<size_t>(size));
return Status::OK();
}
virtual Status Close() override { return Status::OK(); }
virtual Status Flush() override { return Status::OK(); }
virtual Status Sync() override { return Status::OK(); }
virtual Status Append(const Slice& slice) override {
contents_->append(slice.data(), slice.size());
return Status::OK();
}
private: const std::string& GetContent(const std::string& f) { return files_[f]; }
std::string* contents_;
};
explicit StringEnv(Env* t) : EnvWrapper(t) {} const IOStatus WriteToNewFile(const std::string& file_name,
~StringEnv() override {}
const std::string& GetContent(const std::string& f) { return files_[f]; }
const Status WriteToNewFile(const std::string& file_name,
const std::string& content) { const std::string& content) {
std::unique_ptr<WritableFile> r; std::unique_ptr<FSWritableFile> r;
auto s = NewWritableFile(file_name, &r, EnvOptions()); FileOptions file_opts;
if (s.ok()) { IOOptions io_opts;
s = r->Append(content);
}
if (s.ok()) {
s = r->Flush();
}
if (s.ok()) {
s = r->Close();
}
assert(!s.ok() || files_[file_name] == content);
return s;
}
// The following text is boilerplate that forwards all methods to target() auto s = NewWritableFile(file_name, file_opts, &r, nullptr);
Status NewSequentialFile(const std::string& f, if (s.ok()) {
std::unique_ptr<SequentialFile>* r, s = r->Append(content, io_opts, nullptr);
const EnvOptions& /*options*/) override {
auto iter = files_.find(f);
if (iter == files_.end()) {
return Status::NotFound("The specified file does not exist", f);
}
r->reset(new SeqStringSource(iter->second, &num_seq_file_read_));
return Status::OK();
} }
Status NewRandomAccessFile(const std::string& /*f*/, if (s.ok()) {
std::unique_ptr<RandomAccessFile>* /*r*/, s = r->Flush(io_opts, nullptr);
const EnvOptions& /*options*/) override {
return Status::NotSupported();
} }
Status NewWritableFile(const std::string& f, if (s.ok()) {
std::unique_ptr<WritableFile>* r, s = r->Close(io_opts, nullptr);
const EnvOptions& /*options*/) override {
auto iter = files_.find(f);
if (iter != files_.end()) {
return Status::IOError("The specified file already exists", f);
}
r->reset(new StringSink(&files_[f]));
return Status::OK();
}
virtual Status NewDirectory(
const std::string& /*name*/,
std::unique_ptr<Directory>* /*result*/) override {
return Status::NotSupported();
}
Status FileExists(const std::string& f) override {
if (files_.find(f) == files_.end()) {
return Status::NotFound();
}
return Status::OK();
}
Status GetChildren(const std::string& /*dir*/,
std::vector<std::string>* /*r*/) override {
return Status::NotSupported();
}
Status DeleteFile(const std::string& f) override {
files_.erase(f);
return Status::OK();
}
Status CreateDir(const std::string& /*d*/) override {
return Status::NotSupported();
}
Status CreateDirIfMissing(const std::string& /*d*/) override {
return Status::NotSupported();
}
Status DeleteDir(const std::string& /*d*/) override {
return Status::NotSupported();
}
Status GetFileSize(const std::string& f, uint64_t* s) override {
auto iter = files_.find(f);
if (iter == files_.end()) {
return Status::NotFound("The specified file does not exist:", f);
}
*s = iter->second.size();
return Status::OK();
} }
assert(!s.ok() || files_[file_name] == content);
return s;
}
Status GetFileModificationTime(const std::string& /*fname*/, // The following text is boilerplate that forwards all methods to target()
uint64_t* /*file_mtime*/) override { IOStatus NewSequentialFile(const std::string& f,
return Status::NotSupported(); const FileOptions& /*options*/,
std::unique_ptr<FSSequentialFile>* r,
IODebugContext* /*dbg*/) override {
auto iter = files_.find(f);
if (iter == files_.end()) {
return IOStatus::NotFound("The specified file does not exist", f);
} }
r->reset(new SeqStringSource(iter->second, &num_seq_file_read_));
return IOStatus::OK();
}
Status RenameFile(const std::string& /*s*/, IOStatus NewRandomAccessFile(const std::string& /*f*/,
const std::string& /*t*/) override { const FileOptions& /*options*/,
return Status::NotSupported(); std::unique_ptr<FSRandomAccessFile>* /*r*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus NewWritableFile(const std::string& f, const FileOptions& /*options*/,
std::unique_ptr<FSWritableFile>* r,
IODebugContext* /*dbg*/) override {
auto iter = files_.find(f);
if (iter != files_.end()) {
return IOStatus::IOError("The specified file already exists", f);
} }
r->reset(new StringSink(&files_[f]));
return IOStatus::OK();
}
IOStatus NewDirectory(const std::string& /*name*/,
const IOOptions& /*options*/,
std::unique_ptr<FSDirectory>* /*result*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
Status LinkFile(const std::string& /*s*/, IOStatus FileExists(const std::string& f, const IOOptions& /*options*/,
const std::string& /*t*/) override { IODebugContext* /*dbg*/) override {
return Status::NotSupported(); if (files_.find(f) == files_.end()) {
return IOStatus::NotFound();
} }
return IOStatus::OK();
}
Status LockFile(const std::string& /*f*/, FileLock** /*l*/) override { IOStatus GetChildren(const std::string& /*dir*/, const IOOptions& /*options*/,
return Status::NotSupported(); std::vector<std::string>* /*r*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus DeleteFile(const std::string& f, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
files_.erase(f);
return IOStatus::OK();
}
IOStatus CreateDir(const std::string& /*d*/, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus CreateDirIfMissing(const std::string& /*d*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus DeleteDir(const std::string& /*d*/, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus GetFileSize(const std::string& f, const IOOptions& /*options*/,
uint64_t* s, IODebugContext* /*dbg*/) override {
auto iter = files_.find(f);
if (iter == files_.end()) {
return IOStatus::NotFound("The specified file does not exist:", f);
} }
*s = iter->second.size();
return IOStatus::OK();
}
Status UnlockFile(FileLock* /*l*/) override { IOStatus GetFileModificationTime(const std::string& /*fname*/,
return Status::NotSupported(); const IOOptions& /*options*/,
} uint64_t* /*file_mtime*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
std::atomic<int> num_seq_file_read_; IOStatus RenameFile(const std::string& /*s*/, const std::string& /*t*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
protected: IOStatus LinkFile(const std::string& /*s*/, const std::string& /*t*/,
std::unordered_map<std::string, std::string> files_; const IOOptions& /*options*/,
}; IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus LockFile(const std::string& /*f*/, const IOOptions& /*options*/,
FileLock** /*l*/, IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
IOStatus UnlockFile(FileLock* /*l*/, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return IOStatus::NotSupported();
}
std::atomic<int> num_seq_file_read_;
protected:
std::unordered_map<std::string, std::string> files_;
};
// Randomly initialize the given DBOptions // Randomly initialize the given DBOptions
void RandomInitDBOptions(DBOptions* db_opt, Random* rnd); void RandomInitDBOptions(DBOptions* db_opt, Random* rnd);

20
util/file_reader_writer_test.cc
View File

@ -246,19 +246,21 @@ class ReadaheadRandomAccessFileTest
ReadaheadRandomAccessFileTest() : control_contents_() {} ReadaheadRandomAccessFileTest() : control_contents_() {}
std::string Read(uint64_t offset, size_t n) { std::string Read(uint64_t offset, size_t n) {
Slice result; Slice result;
Status s = test_read_holder_->Read(offset, n, &result, scratch_.get()); Status s = test_read_holder_->Read(offset, n, IOOptions(), &result,
scratch_.get(), nullptr);
EXPECT_TRUE(s.ok() || s.IsInvalidArgument()); EXPECT_TRUE(s.ok() || s.IsInvalidArgument());
return std::string(result.data(), result.size()); return std::string(result.data(), result.size());
} }
void ResetSourceStr(const std::string& str = "") { void ResetSourceStr(const std::string& str = "") {
auto write_holder = std::unique_ptr<FSWritableFile> sink(
std::unique_ptr<WritableFileWriter>(test::GetWritableFileWriter( new test::StringSink(&control_contents_));
new test::StringSink(&control_contents_), "" /* don't care */)); std::unique_ptr<WritableFileWriter> write_holder(new WritableFileWriter(
std::move(sink), "" /* don't care */, FileOptions()));
Status s = write_holder->Append(Slice(str)); Status s = write_holder->Append(Slice(str));
EXPECT_OK(s); EXPECT_OK(s);
s = write_holder->Flush(); s = write_holder->Flush();
EXPECT_OK(s); EXPECT_OK(s);
auto read_holder = std::unique_ptr<RandomAccessFile>( std::unique_ptr<FSRandomAccessFile> read_holder(
new test::StringSource(control_contents_)); new test::StringSource(control_contents_));
test_read_holder_ = test_read_holder_ =
NewReadaheadRandomAccessFile(std::move(read_holder), readahead_size_); NewReadaheadRandomAccessFile(std::move(read_holder), readahead_size_);
@ -268,7 +270,7 @@ class ReadaheadRandomAccessFileTest
private: private:
size_t readahead_size_; size_t readahead_size_;
Slice control_contents_; Slice control_contents_;
std::unique_ptr<RandomAccessFile> test_read_holder_; std::unique_ptr<FSRandomAccessFile> test_read_holder_;
std::unique_ptr<char[]> scratch_; std::unique_ptr<char[]> scratch_;
}; };
@ -353,10 +355,10 @@ class ReadaheadSequentialFileTest : public testing::Test,
} }
void Skip(size_t n) { test_read_holder_->Skip(n); } void Skip(size_t n) { test_read_holder_->Skip(n); }
void ResetSourceStr(const std::string& str = "") { void ResetSourceStr(const std::string& str = "") {
auto read_holder = std::unique_ptr<SequentialFile>( auto read_holder = std::unique_ptr<FSSequentialFile>(
new test::SeqStringSource(str, &seq_read_count_)); new test::SeqStringSource(str, &seq_read_count_));
test_read_holder_.reset(new SequentialFileReader( test_read_holder_.reset(new SequentialFileReader(std::move(read_holder),
NewLegacySequentialFileWrapper(read_holder), "test", readahead_size_)); "test", readahead_size_));
} }
size_t GetReadaheadSize() const { return readahead_size_; } size_t GetReadaheadSize() const { return readahead_size_; }

20
utilities/blob_db/blob_dump_tool.cc
View File

@ -5,17 +5,19 @@
#ifndef ROCKSDB_LITE #ifndef ROCKSDB_LITE
#include "utilities/blob_db/blob_dump_tool.h" #include "utilities/blob_db/blob_dump_tool.h"
#include <stdio.h> #include <stdio.h>
#include <cinttypes> #include <cinttypes>
#include <iostream> #include <iostream>
#include <memory> #include <memory>
#include <string> #include <string>
#include "env/composite_env_wrapper.h"
#include "file/random_access_file_reader.h" #include "file/random_access_file_reader.h"
#include "file/readahead_raf.h" #include "file/readahead_raf.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
#include "rocksdb/env.h" #include "rocksdb/file_system.h"
#include "table/format.h" #include "table/format.h"
#include "util/coding.h" #include "util/coding.h"
#include "util/string_util.h" #include "util/string_util.h"
@ -32,18 +34,19 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
bool show_summary) { bool show_summary) {
constexpr size_t kReadaheadSize = 2 * 1024 * 1024; constexpr size_t kReadaheadSize = 2 * 1024 * 1024;
Status s; Status s;
Env* env = Env::Default(); const auto fs = FileSystem::Default();
s = env->FileExists(filename); IOOptions io_opts;
s = fs->FileExists(filename, io_opts, nullptr);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
uint64_t file_size = 0; uint64_t file_size = 0;
s = env->GetFileSize(filename, &file_size); s = fs->GetFileSize(filename, io_opts, &file_size, nullptr);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
std::unique_ptr<RandomAccessFile> file; std::unique_ptr<FSRandomAccessFile> file;
s = env->NewRandomAccessFile(filename, &file, EnvOptions()); s = fs->NewRandomAccessFile(filename, FileOptions(), &file, nullptr);
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -51,8 +54,7 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
if (file_size == 0) { if (file_size == 0) {
return Status::Corruption("File is empty."); return Status::Corruption("File is empty.");
} }
reader_.reset(new RandomAccessFileReader( reader_.reset(new RandomAccessFileReader(std::move(file), filename));
NewLegacyRandomAccessFileWrapper(file), filename));
uint64_t offset = 0; uint64_t offset = 0;
uint64_t footer_offset = 0; uint64_t footer_offset = 0;
CompressionType compression = kNoCompression; CompressionType compression = kNoCompression;