Create a CustomEnv class; Add WinFileSystem; Make LegacyFileSystemWrapper private (#7703)

Summary:
This PR does the following:
-> Creates a WinFileSystem class.  This class is the Windows equivalent of the PosixFileSystem and will be used on Windows systems.
-> Introduces a CustomEnv class.  A CustomEnv is an Env that takes a FileSystem as constructor argument.  I believe there will only ever be two implementations of this class (PosixEnv and WinEnv).  There is still a CustomEnvWrapper class that takes an Env and a FileSystem and wraps the Env calls with the input Env but uses the FileSystem for the FileSystem calls
-> Eliminates the public uses of the LegacyFileSystemWrapper.

With this change in place, there are effectively the following patterns of Env:
- "Base Env classes" (PosixEnv, WinEnv).  These classes implement the core Env functions (e.g. Threads) and have a hard-coded input FileSystem.  These classes inherit from CompositeEnv, implement the core Env functions (threads) and delegate the FileSystem-like calls to the input file system.
- Wrapped Composite Env classes (MemEnv).  These classes take in an Env and a FileSystem.  The core env functions are re-directed to the wrapped env.  The file system calls are redirected to the input file system
- Legacy Wrapped Env classes.  These classes take in an Env input (but no FileSystem).  The core env functions are re-directed to the wrapped env.  A "Legacy File System" is created using this env and the file system calls directed to the env itself.

With these changes in place, the PosixEnv becomes a singleton -- there is only ever one created.  Any other use of the PosixEnv is via another wrapped env.  This cleans up some of the issues with the env construction and destruction.

Additionally, there were places in the code that required had an Env when they required a FileSystem.  Many of these places would wrap the Env with a LegacyFileSystemWrapper instead of using the env->GetFileSystem().  These places were changed, thereby removing layers of additional redirection (LegacyFileSystem --> Env --> Env::FileSystem).

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7703

Reviewed By: zhichao-cao

Differential Revision: D25762190

Pulled By: anand1976

fbshipit-source-id: 1a088e97fc916f28ac69c149cd1dcad0ab31704b
This commit is contained in:
mrambacher 2021-01-06 10:48:24 -08:00 committed by Facebook GitHub Bot
parent c1a65a4de4
commit e628f59e87
31 changed files with 1498 additions and 1553 deletions

View File

@ -15,7 +15,6 @@
#include "db/blob/blob_index.h" #include "db/blob/blob_index.h"
#include "db/blob/blob_log_format.h" #include "db/blob/blob_log_format.h"
#include "db/blob/blob_log_sequential_reader.h" #include "db/blob/blob_log_sequential_reader.h"
#include "env/composite_env_wrapper.h"
#include "env/mock_env.h" #include "env/mock_env.h"
#include "file/filename.h" #include "file/filename.h"
#include "file/random_access_file_reader.h" #include "file/random_access_file_reader.h"
@ -40,7 +39,9 @@ class TestFileNumberGenerator {
class BlobFileBuilderTest : public testing::Test { class BlobFileBuilderTest : public testing::Test {
protected: protected:
BlobFileBuilderTest() : mock_env_(Env::Default()), fs_(&mock_env_) {} BlobFileBuilderTest() : mock_env_(Env::Default()) {
fs_ = mock_env_.GetFileSystem();
}
void VerifyBlobFile(uint64_t blob_file_number, void VerifyBlobFile(uint64_t blob_file_number,
const std::string& blob_file_path, const std::string& blob_file_path,
@ -54,7 +55,7 @@ class BlobFileBuilderTest : public testing::Test {
std::unique_ptr<FSRandomAccessFile> file; std::unique_ptr<FSRandomAccessFile> file;
constexpr IODebugContext* dbg = nullptr; constexpr IODebugContext* dbg = nullptr;
ASSERT_OK( ASSERT_OK(
fs_.NewRandomAccessFile(blob_file_path, file_options_, &file, dbg)); fs_->NewRandomAccessFile(blob_file_path, file_options_, &file, dbg));
std::unique_ptr<RandomAccessFileReader> file_reader( std::unique_ptr<RandomAccessFileReader> file_reader(
new RandomAccessFileReader(std::move(file), blob_file_path, new RandomAccessFileReader(std::move(file), blob_file_path,
@ -108,7 +109,7 @@ class BlobFileBuilderTest : public testing::Test {
} }
MockEnv mock_env_; MockEnv mock_env_;
LegacyFileSystemWrapper fs_; std::shared_ptr<FileSystem> fs_;
FileOptions file_options_; FileOptions file_options_;
}; };
@ -138,7 +139,7 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_.get(),
&immutable_cf_options, &mutable_cf_options, &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
@ -221,7 +222,7 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_.get(),
&immutable_cf_options, &mutable_cf_options, &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
@ -306,7 +307,7 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_.get(),
&immutable_cf_options, &mutable_cf_options, &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
@ -358,7 +359,7 @@ TEST_F(BlobFileBuilderTest, Compression) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_.get(),
&immutable_cf_options, &mutable_cf_options, &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
@ -440,7 +441,7 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_.get(),
&immutable_cf_options, &mutable_cf_options, &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
@ -517,7 +518,7 @@ TEST_F(BlobFileBuilderTest, Checksum) {
std::vector<std::string> blob_file_paths; std::vector<std::string> blob_file_paths;
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, &fs_, BlobFileBuilder builder(TestFileNumberGenerator(), &mock_env_, fs_.get(),
&immutable_cf_options, &mutable_cf_options, &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
@ -571,12 +572,12 @@ class BlobFileBuilderIOErrorTest
BlobFileBuilderIOErrorTest() BlobFileBuilderIOErrorTest()
: mock_env_(Env::Default()), : mock_env_(Env::Default()),
fault_injection_env_(&mock_env_), fault_injection_env_(&mock_env_),
fs_(&fault_injection_env_), fs_(fault_injection_env_.GetFileSystem()),
sync_point_(GetParam()) {} sync_point_(GetParam()) {}
MockEnv mock_env_; MockEnv mock_env_;
FaultInjectionTestEnv fault_injection_env_; FaultInjectionTestEnv fault_injection_env_;
LegacyFileSystemWrapper fs_; std::shared_ptr<FileSystem> fs_;
FileOptions file_options_; FileOptions file_options_;
std::string sync_point_; std::string sync_point_;
}; };
@ -616,7 +617,7 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
std::vector<BlobFileAddition> blob_file_additions; std::vector<BlobFileAddition> blob_file_additions;
BlobFileBuilder builder(TestFileNumberGenerator(), &fault_injection_env_, BlobFileBuilder builder(TestFileNumberGenerator(), &fault_injection_env_,
&fs_, &immutable_cf_options, &mutable_cf_options, fs_.get(), &immutable_cf_options, &mutable_cf_options,
&file_options_, job_id, column_family_id, &file_options_, job_id, column_family_id,
column_family_name, io_priority, write_hint, column_family_name, io_priority, write_hint,
&blob_file_paths, &blob_file_additions); &blob_file_paths, &blob_file_additions);

View File

@ -72,7 +72,7 @@ class CompactionJobTestBase : public testing::Test {
CompactionJobTestBase(std::string dbname, const Comparator* ucmp, CompactionJobTestBase(std::string dbname, const Comparator* ucmp,
std::function<std::string(uint64_t)> encode_u64_ts) std::function<std::string(uint64_t)> encode_u64_ts)
: env_(Env::Default()), : env_(Env::Default()),
fs_(std::make_shared<LegacyFileSystemWrapper>(env_)), fs_(env_->GetFileSystem()),
dbname_(std::move(dbname)), dbname_(std::move(dbname)),
ucmp_(ucmp), ucmp_(ucmp),
db_options_(), db_options_(),

View File

@ -3274,7 +3274,9 @@ class DeadlineFS : public FileSystemWrapper {
// Increment the IO counter and return a delay in microseconds // Increment the IO counter and return a delay in microseconds
IOStatus ShouldDelay(const IOOptions& opts) { IOStatus ShouldDelay(const IOOptions& opts) {
if (!deadline_.count() && !io_timeout_.count()) { if (timedout_) {
return IOStatus::TimedOut();
} else if (!deadline_.count() && !io_timeout_.count()) {
return IOStatus::OK(); return IOStatus::OK();
} }
if (!ignore_deadline_ && delay_trigger_ == io_count_++) { if (!ignore_deadline_ && delay_trigger_ == io_count_++) {

View File

@ -53,7 +53,6 @@
#include "db/version_set.h" #include "db/version_set.h"
#include "db/write_batch_internal.h" #include "db/write_batch_internal.h"
#include "db/write_callback.h" #include "db/write_callback.h"
#include "env/composite_env_wrapper.h"
#include "file/file_util.h" #include "file/file_util.h"
#include "file/filename.h" #include "file/filename.h"
#include "file/random_access_file_reader.h" #include "file/random_access_file_reader.h"
@ -3099,8 +3098,8 @@ const std::string& DBImpl::GetName() const { return dbname_; }
Env* DBImpl::GetEnv() const { return env_; } Env* DBImpl::GetEnv() const { return env_; }
FileSystem* DB::GetFileSystem() const { FileSystem* DB::GetFileSystem() const {
static LegacyFileSystemWrapper fs_wrap(GetEnv()); const auto& fs = GetEnv()->GetFileSystem();
return &fs_wrap; return fs.get();
} }
FileSystem* DBImpl::GetFileSystem() const { FileSystem* DBImpl::GetFileSystem() const {

View File

@ -3685,20 +3685,26 @@ TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) {
TEST_F(DBTest2, TestNumPread) { TEST_F(DBTest2, TestNumPread) {
Options options = CurrentOptions(); Options options = CurrentOptions();
bool prefetch_supported =
test::IsPrefetchSupported(env_->GetFileSystem(), dbname_);
// disable block cache // disable block cache
BlockBasedTableOptions table_options; BlockBasedTableOptions table_options;
table_options.no_block_cache = true; table_options.no_block_cache = true;
options.table_factory.reset(NewBlockBasedTableFactory(table_options)); options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options); Reopen(options);
env_->count_random_reads_ = true; env_->count_random_reads_ = true;
env_->random_file_open_counter_.store(0); env_->random_file_open_counter_.store(0);
ASSERT_OK(Put("bar", "foo")); ASSERT_OK(Put("bar", "foo"));
ASSERT_OK(Put("foo", "bar")); ASSERT_OK(Put("foo", "bar"));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
if (prefetch_supported) {
// After flush, we'll open the file and read footer, meta block, // After flush, we'll open the file and read footer, meta block,
// property block and index block. // property block and index block.
ASSERT_EQ(4, env_->random_read_counter_.Read()); ASSERT_EQ(4, env_->random_read_counter_.Read());
} else {
// With prefetch not supported, we will do a single read into a buffer
ASSERT_EQ(1, env_->random_read_counter_.Read());
}
ASSERT_EQ(1, env_->random_file_open_counter_.load()); ASSERT_EQ(1, env_->random_file_open_counter_.load());
// One pread per a normal data block read // One pread per a normal data block read
@ -3714,19 +3720,30 @@ TEST_F(DBTest2, TestNumPread) {
ASSERT_OK(Put("bar2", "foo2")); ASSERT_OK(Put("bar2", "foo2"));
ASSERT_OK(Put("foo2", "bar2")); ASSERT_OK(Put("foo2", "bar2"));
ASSERT_OK(Flush()); ASSERT_OK(Flush());
if (prefetch_supported) {
// After flush, we'll open the file and read footer, meta block, // After flush, we'll open the file and read footer, meta block,
// property block and index block. // property block and index block.
ASSERT_EQ(4, env_->random_read_counter_.Read()); ASSERT_EQ(4, env_->random_read_counter_.Read());
} else {
// With prefetch not supported, we will do a single read into a buffer
ASSERT_EQ(1, env_->random_read_counter_.Read());
}
ASSERT_EQ(1, env_->random_file_open_counter_.load()); ASSERT_EQ(1, env_->random_file_open_counter_.load());
// Compaction needs two input blocks, which requires 2 preads, and
// generate a new SST file which needs 4 preads (footer, meta block,
// property block and index block). In total 6.
env_->random_file_open_counter_.store(0); env_->random_file_open_counter_.store(0);
env_->random_read_counter_.Reset(); env_->random_read_counter_.Reset();
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
if (prefetch_supported) {
// Compaction needs two input blocks, which requires 2 preads, and
// generate a new SST file which needs 4 preads (footer, meta block,
// property block and index block). In total 6.
ASSERT_EQ(6, env_->random_read_counter_.Read()); ASSERT_EQ(6, env_->random_read_counter_.Read());
// All compactin input files should have already been opened. } else {
// With prefetch off, compaction needs two input blocks,
// followed by a single buffered read. In total 3.
ASSERT_EQ(3, env_->random_read_counter_.Read());
}
// All compaction input files should have already been opened.
ASSERT_EQ(1, env_->random_file_open_counter_.load()); ASSERT_EQ(1, env_->random_file_open_counter_.load());
// One pread per a normal data block read // One pread per a normal data block read

View File

@ -32,7 +32,7 @@ class FlushJobTestBase : public testing::Test {
protected: protected:
FlushJobTestBase(std::string dbname, const Comparator* ucmp) FlushJobTestBase(std::string dbname, const Comparator* ucmp)
: env_(Env::Default()), : env_(Env::Default()),
fs_(std::make_shared<LegacyFileSystemWrapper>(env_)), fs_(env_->GetFileSystem()),
dbname_(std::move(dbname)), dbname_(std::move(dbname)),
ucmp_(ucmp), ucmp_(ucmp),
options_(), options_(),

View File

@ -439,7 +439,6 @@ class Repairer {
range_del_iters.emplace_back(range_del_iter); range_del_iters.emplace_back(range_del_iter);
} }
LegacyFileSystemWrapper fs(env_);
IOStatus io_s; IOStatus io_s;
status = BuildTable( status = BuildTable(
dbname_, /* versions */ nullptr, immutable_db_options_, dbname_, /* versions */ nullptr, immutable_db_options_,

View File

@ -80,8 +80,8 @@ TEST_F(RepairTest, CorruptManifest) {
Close(); Close();
ASSERT_OK(env_->FileExists(manifest_path)); ASSERT_OK(env_->FileExists(manifest_path));
LegacyFileSystemWrapper fs(env_); ASSERT_OK(CreateFile(env_->GetFileSystem(), manifest_path, "blah",
ASSERT_OK(CreateFile(&fs, manifest_path, "blah", false /* use_fsync */)); false /* use_fsync */));
ASSERT_OK(RepairDB(dbname_, CurrentOptions())); ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
Reopen(CurrentOptions()); Reopen(CurrentOptions());
@ -163,8 +163,8 @@ TEST_F(RepairTest, CorruptSst) {
ASSERT_OK(GetFirstSstPath(&sst_path)); ASSERT_OK(GetFirstSstPath(&sst_path));
ASSERT_FALSE(sst_path.empty()); ASSERT_FALSE(sst_path.empty());
LegacyFileSystemWrapper fs(env_); ASSERT_OK(CreateFile(env_->GetFileSystem(), sst_path, "blah",
ASSERT_OK(CreateFile(&fs, sst_path, "blah", false /* use_fsync */)); false /* use_fsync */));
Close(); Close();
ASSERT_OK(RepairDB(dbname_, CurrentOptions())); ASSERT_OK(RepairDB(dbname_, CurrentOptions()));

View File

@ -47,8 +47,7 @@ class WalManagerTest : public testing::Test {
std::numeric_limits<uint64_t>::max()); std::numeric_limits<uint64_t>::max());
db_options_.wal_dir = dbname_; db_options_.wal_dir = dbname_;
db_options_.env = env_.get(); db_options_.env = env_.get();
fs_.reset(new LegacyFileSystemWrapper(env_.get())); db_options_.fs = env_->GetFileSystem();
db_options_.fs = fs_;
versions_.reset( versions_.reset(
new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(), new VersionSet(dbname_, &db_options_, env_options_, table_cache_.get(),
@ -116,7 +115,6 @@ class WalManagerTest : public testing::Test {
WriteBufferManager write_buffer_manager_; WriteBufferManager write_buffer_manager_;
std::unique_ptr<VersionSet> versions_; std::unique_ptr<VersionSet> versions_;
std::unique_ptr<WalManager> wal_manager_; std::unique_ptr<WalManager> wal_manager_;
std::shared_ptr<LegacyFileSystemWrapper> fs_;
std::unique_ptr<log::Writer> current_log_writer_; std::unique_ptr<log::Writer> current_log_writer_;
uint64_t current_log_number_; uint64_t current_log_number_;

View File

@ -8,6 +8,13 @@
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/file_system.h" #include "rocksdb/file_system.h"
#ifdef _WIN32
// Windows API macro interference
#undef DeleteFile
#undef GetCurrentTime
#undef LoadLibrary
#endif
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
// The CompositeEnvWrapper class provides an interface that is compatible // The CompositeEnvWrapper class provides an interface that is compatible
@ -271,16 +278,11 @@ class CompositeDirectoryWrapper : public Directory {
std::unique_ptr<FSDirectory> target_; std::unique_ptr<FSDirectory> target_;
}; };
class CompositeEnvWrapper : public Env { class CompositeEnv : public Env {
public: public:
// Initialize a CompositeEnvWrapper that delegates all thread/time related // Initialize a CompositeEnvWrapper that delegates all thread/time related
// calls to env, and all file operations to fs // calls to env, and all file operations to fs
explicit CompositeEnvWrapper(Env* env, std::shared_ptr<FileSystem> fs) explicit CompositeEnv(const std::shared_ptr<FileSystem>& fs) : Env(fs) {}
: Env(fs), env_target_(env) {}
~CompositeEnvWrapper() {}
// Return the target to which this Env forwards all calls
Env* env_target() const { return env_target_; }
Status RegisterDbPaths(const std::vector<std::string>& paths) override { Status RegisterDbPaths(const std::vector<std::string>& paths) override {
return file_system_->RegisterDbPaths(paths); return file_system_->RegisterDbPaths(paths);
@ -498,6 +500,66 @@ class CompositeEnvWrapper : public Env {
return file_system_->IsDirectory(path, io_opts, is_dir, &dbg); return file_system_->IsDirectory(path, io_opts, is_dir, &dbg);
} }
Status GetTestDirectory(std::string* path) override {
IOOptions io_opts;
IODebugContext dbg;
return file_system_->GetTestDirectory(io_opts, path, &dbg);
}
EnvOptions OptimizeForLogRead(const EnvOptions& env_options) const override {
return file_system_->OptimizeForLogRead(FileOptions(env_options));
}
EnvOptions OptimizeForManifestRead(
const EnvOptions& env_options) const override {
return file_system_->OptimizeForManifestRead(FileOptions(env_options));
}
EnvOptions OptimizeForLogWrite(const EnvOptions& env_options,
const DBOptions& db_options) const override {
return file_system_->OptimizeForLogWrite(FileOptions(env_options),
db_options);
}
EnvOptions OptimizeForManifestWrite(
const EnvOptions& env_options) const override {
return file_system_->OptimizeForManifestWrite(FileOptions(env_options));
}
EnvOptions OptimizeForCompactionTableWrite(
const EnvOptions& env_options,
const ImmutableDBOptions& immutable_ops) const override {
return file_system_->OptimizeForCompactionTableWrite(
FileOptions(env_options), immutable_ops);
}
EnvOptions OptimizeForCompactionTableRead(
const EnvOptions& env_options,
const ImmutableDBOptions& db_options) const override {
return file_system_->OptimizeForCompactionTableRead(
FileOptions(env_options), db_options);
}
// This seems to clash with a macro on Windows, so #undef it here
#ifdef GetFreeSpace
#undef GetFreeSpace
#endif
Status GetFreeSpace(const std::string& path, uint64_t* diskfree) override {
IOOptions io_opts;
IODebugContext dbg;
return file_system_->GetFreeSpace(path, io_opts, diskfree, &dbg);
}
};
class CompositeEnvWrapper : public CompositeEnv {
public:
// Initialize a CompositeEnvWrapper that delegates all thread/time related
// calls to env, and all file operations to fs
explicit CompositeEnvWrapper(Env* env, const std::shared_ptr<FileSystem>& fs)
: CompositeEnv(fs), env_target_(env) {}
// Return the target to which this Env forwards all calls
Env* env_target() const { return env_target_; }
#if !defined(OS_WIN) && !defined(ROCKSDB_NO_DYNAMIC_EXTENSION) #if !defined(OS_WIN) && !defined(ROCKSDB_NO_DYNAMIC_EXTENSION)
Status LoadLibrary(const std::string& lib_name, Status LoadLibrary(const std::string& lib_name,
const std::string& search_path, const std::string& search_path,
@ -522,11 +584,7 @@ class CompositeEnvWrapper : public Env {
unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override { unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override {
return env_target_->GetThreadPoolQueueLen(pri); return env_target_->GetThreadPoolQueueLen(pri);
} }
Status GetTestDirectory(std::string* path) override {
IOOptions io_opts;
IODebugContext dbg;
return file_system_->GetTestDirectory(io_opts, path, &dbg);
}
uint64_t NowMicros() override { return env_target_->NowMicros(); } uint64_t NowMicros() override { return env_target_->NowMicros(); }
uint64_t NowNanos() override { return env_target_->NowNanos(); } uint64_t NowNanos() override { return env_target_->NowNanos(); }
uint64_t NowCPUNanos() override { return env_target_->NowCPUNanos(); } uint64_t NowCPUNanos() override { return env_target_->NowCPUNanos(); }
@ -585,45 +643,6 @@ class CompositeEnvWrapper : public Env {
return env_target_->GenerateUniqueId(); return env_target_->GenerateUniqueId();
} }
EnvOptions OptimizeForLogRead(const EnvOptions& env_options) const override {
return file_system_->OptimizeForLogRead(FileOptions(env_options));
}
EnvOptions OptimizeForManifestRead(
const EnvOptions& env_options) const override {
return file_system_->OptimizeForManifestRead(FileOptions(env_options));
}
EnvOptions OptimizeForLogWrite(const EnvOptions& env_options,
const DBOptions& db_options) const override {
return file_system_->OptimizeForLogWrite(FileOptions(env_options),
db_options);
}
EnvOptions OptimizeForManifestWrite(
const EnvOptions& env_options) const override {
return file_system_->OptimizeForManifestWrite(FileOptions(env_options));
}
EnvOptions OptimizeForCompactionTableWrite(
const EnvOptions& env_options,
const ImmutableDBOptions& immutable_ops) const override {
return file_system_->OptimizeForCompactionTableWrite(
FileOptions(env_options), immutable_ops);
}
EnvOptions OptimizeForCompactionTableRead(
const EnvOptions& env_options,
const ImmutableDBOptions& db_options) const override {
return file_system_->OptimizeForCompactionTableRead(
FileOptions(env_options), db_options);
}
// This seems to clash with a macro on Windows, so #undef it here
#ifdef GetFreeSpace
#undef GetFreeSpace
#endif
Status GetFreeSpace(const std::string& path, uint64_t* diskfree) override {
IOOptions io_opts;
IODebugContext dbg;
return file_system_->GetFreeSpace(path, io_opts, diskfree, &dbg);
}
private: private:
Env* env_target_; Env* env_target_;
}; };
@ -880,249 +899,6 @@ class LegacyDirectoryWrapper : public FSDirectory {
std::unique_ptr<Directory> target_; std::unique_ptr<Directory> target_;
}; };
class LegacyFileSystemWrapper : public FileSystem {
public:
// Initialize an EnvWrapper that delegates all calls to *t
explicit LegacyFileSystemWrapper(Env* t) : target_(t) {}
~LegacyFileSystemWrapper() override {}
const char* Name() const override { return "Legacy File System"; }
// Return the target to which this Env forwards all calls
Env* target() const { return target_; }
// The following text is boilerplate that forwards all methods to target()
IOStatus NewSequentialFile(const std::string& f,
const FileOptions& file_opts,
std::unique_ptr<FSSequentialFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<SequentialFile> file;
Status s = target_->NewSequentialFile(f, &file, file_opts);
if (s.ok()) {
r->reset(new LegacySequentialFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewRandomAccessFile(const std::string& f,
const FileOptions& file_opts,
std::unique_ptr<FSRandomAccessFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<RandomAccessFile> file;
Status s = target_->NewRandomAccessFile(f, &file, file_opts);
if (s.ok()) {
r->reset(new LegacyRandomAccessFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<WritableFile> file;
Status s = target_->NewWritableFile(f, &file, file_opts);
if (s.ok()) {
r->reset(new LegacyWritableFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus ReopenWritableFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* /*dbg*/) override {
std::unique_ptr<WritableFile> file;
Status s = target_->ReopenWritableFile(fname, &file, file_opts);
if (s.ok()) {
result->reset(new LegacyWritableFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<WritableFile> file;
Status s = target_->ReuseWritableFile(fname, old_fname, &file, file_opts);
if (s.ok()) {
r->reset(new LegacyWritableFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewRandomRWFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* /*dbg*/) override {
std::unique_ptr<RandomRWFile> file;
Status s = target_->NewRandomRWFile(fname, &file, file_opts);
if (s.ok()) {
result->reset(new LegacyRandomRWFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewMemoryMappedFileBuffer(
const std::string& fname,
std::unique_ptr<MemoryMappedFileBuffer>* result) override {
return status_to_io_status(
target_->NewMemoryMappedFileBuffer(fname, result));
}
IOStatus NewDirectory(const std::string& name, const IOOptions& /*io_opts*/,
std::unique_ptr<FSDirectory>* result,
IODebugContext* /*dbg*/) override {
std::unique_ptr<Directory> dir;
Status s = target_->NewDirectory(name, &dir);
if (s.ok()) {
result->reset(new LegacyDirectoryWrapper(std::move(dir)));
}
return status_to_io_status(std::move(s));
}
IOStatus FileExists(const std::string& f, const IOOptions& /*io_opts*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->FileExists(f));
}
IOStatus GetChildren(const std::string& dir, const IOOptions& /*io_opts*/,
std::vector<std::string>* r,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetChildren(dir, r));
}
IOStatus GetChildrenFileAttributes(const std::string& dir,
const IOOptions& /*options*/,
std::vector<FileAttributes>* result,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetChildrenFileAttributes(dir, result));
}
IOStatus DeleteFile(const std::string& f, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->DeleteFile(f));
}
IOStatus Truncate(const std::string& fname, size_t size,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->Truncate(fname, size));
}
IOStatus CreateDir(const std::string& d, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->CreateDir(d));
}
IOStatus CreateDirIfMissing(const std::string& d,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->CreateDirIfMissing(d));
}
IOStatus DeleteDir(const std::string& d, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->DeleteDir(d));
}
IOStatus GetFileSize(const std::string& f, const IOOptions& /*options*/,
uint64_t* s, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetFileSize(f, s));
}
IOStatus GetFileModificationTime(const std::string& fname,
const IOOptions& /*options*/,
uint64_t* file_mtime,
IODebugContext* /*dbg*/) override {
return status_to_io_status(
target_->GetFileModificationTime(fname, file_mtime));
}
IOStatus GetAbsolutePath(const std::string& db_path,
const IOOptions& /*options*/,
std::string* output_path,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetAbsolutePath(db_path, output_path));
}
IOStatus RenameFile(const std::string& s, const std::string& t,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->RenameFile(s, t));
}
IOStatus LinkFile(const std::string& s, const std::string& t,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->LinkFile(s, t));
}
IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*options*/,
uint64_t* count, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->NumFileLinks(fname, count));
}
IOStatus AreFilesSame(const std::string& first, const std::string& second,
const IOOptions& /*options*/, bool* res,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->AreFilesSame(first, second, res));
}
IOStatus LockFile(const std::string& f, const IOOptions& /*options*/,
FileLock** l, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->LockFile(f, l));
}
IOStatus UnlockFile(FileLock* l, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->UnlockFile(l));
}
IOStatus GetTestDirectory(const IOOptions& /*options*/, std::string* path,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetTestDirectory(path));
}
IOStatus NewLogger(const std::string& fname, const IOOptions& /*options*/,
std::shared_ptr<Logger>* result,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->NewLogger(fname, result));
}
void SanitizeFileOptions(FileOptions* opts) const override {
target_->SanitizeEnvOptions(opts);
}
FileOptions OptimizeForLogRead(
const FileOptions& file_options) const override {
return target_->OptimizeForLogRead(file_options);
}
FileOptions OptimizeForManifestRead(
const FileOptions& file_options) const override {
return target_->OptimizeForManifestRead(file_options);
}
FileOptions OptimizeForLogWrite(const FileOptions& file_options,
const DBOptions& db_options) const override {
return target_->OptimizeForLogWrite(file_options, db_options);
}
FileOptions OptimizeForManifestWrite(
const FileOptions& file_options) const override {
return target_->OptimizeForManifestWrite(file_options);
}
FileOptions OptimizeForCompactionTableWrite(
const FileOptions& file_options,
const ImmutableDBOptions& immutable_ops) const override {
return target_->OptimizeForCompactionTableWrite(file_options,
immutable_ops);
}
FileOptions OptimizeForCompactionTableRead(
const FileOptions& file_options,
const ImmutableDBOptions& db_options) const override {
return target_->OptimizeForCompactionTableRead(file_options, db_options);
}
// This seems to clash with a macro on Windows, so #undef it here
#ifdef GetFreeSpace
#undef GetFreeSpace
#endif
IOStatus GetFreeSpace(const std::string& path, const IOOptions& /*options*/,
uint64_t* diskfree, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetFreeSpace(path, diskfree));
}
IOStatus IsDirectory(const std::string& path, const IOOptions& /*options*/,
bool* is_dir, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->IsDirectory(path, is_dir));
}
private:
Env* target_;
};
inline std::unique_ptr<FSSequentialFile> NewLegacySequentialFileWrapper( inline std::unique_ptr<FSSequentialFile> NewLegacySequentialFileWrapper(
std::unique_ptr<SequentialFile>& file) { std::unique_ptr<SequentialFile>& file) {
return std::unique_ptr<FSSequentialFile>( return std::unique_ptr<FSSequentialFile>(

257
env/env.cc vendored
View File

@ -20,6 +20,248 @@
#include "util/autovector.h" #include "util/autovector.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace {
class LegacyFileSystemWrapper : public FileSystem {
public:
// Initialize an EnvWrapper that delegates all calls to *t
explicit LegacyFileSystemWrapper(Env* t) : target_(t) {}
~LegacyFileSystemWrapper() override {}
const char* Name() const override { return "Legacy File System"; }
// Return the target to which this Env forwards all calls
Env* target() const { return target_; }
// The following text is boilerplate that forwards all methods to target()
IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts,
std::unique_ptr<FSSequentialFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<SequentialFile> file;
Status s = target_->NewSequentialFile(f, &file, file_opts);
if (s.ok()) {
r->reset(new LegacySequentialFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewRandomAccessFile(const std::string& f,
const FileOptions& file_opts,
std::unique_ptr<FSRandomAccessFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<RandomAccessFile> file;
Status s = target_->NewRandomAccessFile(f, &file, file_opts);
if (s.ok()) {
r->reset(new LegacyRandomAccessFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<WritableFile> file;
Status s = target_->NewWritableFile(f, &file, file_opts);
if (s.ok()) {
r->reset(new LegacyWritableFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus ReopenWritableFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* /*dbg*/) override {
std::unique_ptr<WritableFile> file;
Status s = target_->ReopenWritableFile(fname, &file, file_opts);
if (s.ok()) {
result->reset(new LegacyWritableFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus ReuseWritableFile(const std::string& fname,
const std::string& old_fname,
const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* r,
IODebugContext* /*dbg*/) override {
std::unique_ptr<WritableFile> file;
Status s = target_->ReuseWritableFile(fname, old_fname, &file, file_opts);
if (s.ok()) {
r->reset(new LegacyWritableFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewRandomRWFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSRandomRWFile>* result,
IODebugContext* /*dbg*/) override {
std::unique_ptr<RandomRWFile> file;
Status s = target_->NewRandomRWFile(fname, &file, file_opts);
if (s.ok()) {
result->reset(new LegacyRandomRWFileWrapper(std::move(file)));
}
return status_to_io_status(std::move(s));
}
IOStatus NewMemoryMappedFileBuffer(
const std::string& fname,
std::unique_ptr<MemoryMappedFileBuffer>* result) override {
return status_to_io_status(
target_->NewMemoryMappedFileBuffer(fname, result));
}
IOStatus NewDirectory(const std::string& name, const IOOptions& /*io_opts*/,
std::unique_ptr<FSDirectory>* result,
IODebugContext* /*dbg*/) override {
std::unique_ptr<Directory> dir;
Status s = target_->NewDirectory(name, &dir);
if (s.ok()) {
result->reset(new LegacyDirectoryWrapper(std::move(dir)));
}
return status_to_io_status(std::move(s));
}
IOStatus FileExists(const std::string& f, const IOOptions& /*io_opts*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->FileExists(f));
}
IOStatus GetChildren(const std::string& dir, const IOOptions& /*io_opts*/,
std::vector<std::string>* r,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetChildren(dir, r));
}
IOStatus GetChildrenFileAttributes(const std::string& dir,
const IOOptions& /*options*/,
std::vector<FileAttributes>* result,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetChildrenFileAttributes(dir, result));
}
IOStatus DeleteFile(const std::string& f, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->DeleteFile(f));
}
IOStatus Truncate(const std::string& fname, size_t size,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->Truncate(fname, size));
}
IOStatus CreateDir(const std::string& d, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->CreateDir(d));
}
IOStatus CreateDirIfMissing(const std::string& d,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->CreateDirIfMissing(d));
}
IOStatus DeleteDir(const std::string& d, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->DeleteDir(d));
}
IOStatus GetFileSize(const std::string& f, const IOOptions& /*options*/,
uint64_t* s, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetFileSize(f, s));
}
IOStatus GetFileModificationTime(const std::string& fname,
const IOOptions& /*options*/,
uint64_t* file_mtime,
IODebugContext* /*dbg*/) override {
return status_to_io_status(
target_->GetFileModificationTime(fname, file_mtime));
}
IOStatus GetAbsolutePath(const std::string& db_path,
const IOOptions& /*options*/,
std::string* output_path,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetAbsolutePath(db_path, output_path));
}
IOStatus RenameFile(const std::string& s, const std::string& t,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->RenameFile(s, t));
}
IOStatus LinkFile(const std::string& s, const std::string& t,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->LinkFile(s, t));
}
IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*options*/,
uint64_t* count, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->NumFileLinks(fname, count));
}
IOStatus AreFilesSame(const std::string& first, const std::string& second,
const IOOptions& /*options*/, bool* res,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->AreFilesSame(first, second, res));
}
IOStatus LockFile(const std::string& f, const IOOptions& /*options*/,
FileLock** l, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->LockFile(f, l));
}
IOStatus UnlockFile(FileLock* l, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->UnlockFile(l));
}
IOStatus GetTestDirectory(const IOOptions& /*options*/, std::string* path,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetTestDirectory(path));
}
IOStatus NewLogger(const std::string& fname, const IOOptions& /*options*/,
std::shared_ptr<Logger>* result,
IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->NewLogger(fname, result));
}
void SanitizeFileOptions(FileOptions* opts) const override {
target_->SanitizeEnvOptions(opts);
}
FileOptions OptimizeForLogRead(
const FileOptions& file_options) const override {
return target_->OptimizeForLogRead(file_options);
}
FileOptions OptimizeForManifestRead(
const FileOptions& file_options) const override {
return target_->OptimizeForManifestRead(file_options);
}
FileOptions OptimizeForLogWrite(const FileOptions& file_options,
const DBOptions& db_options) const override {
return target_->OptimizeForLogWrite(file_options, db_options);
}
FileOptions OptimizeForManifestWrite(
const FileOptions& file_options) const override {
return target_->OptimizeForManifestWrite(file_options);
}
FileOptions OptimizeForCompactionTableWrite(
const FileOptions& file_options,
const ImmutableDBOptions& immutable_ops) const override {
return target_->OptimizeForCompactionTableWrite(file_options,
immutable_ops);
}
FileOptions OptimizeForCompactionTableRead(
const FileOptions& file_options,
const ImmutableDBOptions& db_options) const override {
return target_->OptimizeForCompactionTableRead(file_options, db_options);
}
#ifdef GetFreeSpace
#undef GetFreeSpace
#endif
IOStatus GetFreeSpace(const std::string& path, const IOOptions& /*options*/,
uint64_t* diskfree, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->GetFreeSpace(path, diskfree));
}
IOStatus IsDirectory(const std::string& path, const IOOptions& /*options*/,
bool* is_dir, IODebugContext* /*dbg*/) override {
return status_to_io_status(target_->IsDirectory(path, is_dir));
}
private:
Env* target_;
};
} // end anonymous namespace
Env::Env() : thread_status_updater_(nullptr) { Env::Env() : thread_status_updater_(nullptr) {
file_system_ = std::make_shared<LegacyFileSystemWrapper>(this); file_system_ = std::make_shared<LegacyFileSystemWrapper>(this);
@ -386,13 +628,13 @@ void Log(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
Status WriteStringToFile(Env* env, const Slice& data, const std::string& fname, Status WriteStringToFile(Env* env, const Slice& data, const std::string& fname,
bool should_sync) { bool should_sync) {
LegacyFileSystemWrapper lfsw(env); const auto& fs = env->GetFileSystem();
return WriteStringToFile(&lfsw, data, fname, should_sync); return WriteStringToFile(fs.get(), data, fname, should_sync);
} }
Status ReadFileToString(Env* env, const std::string& fname, std::string* data) { Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
LegacyFileSystemWrapper lfsw(env); const auto& fs = env->GetFileSystem();
return ReadFileToString(&lfsw, fname, data); return ReadFileToString(fs.get(), fname, data);
} }
EnvWrapper::~EnvWrapper() { EnvWrapper::~EnvWrapper() {
@ -488,11 +730,4 @@ Status NewEnvLogger(const std::string& fname, Env* env,
const std::shared_ptr<FileSystem>& Env::GetFileSystem() const { const std::shared_ptr<FileSystem>& Env::GetFileSystem() const {
return file_system_; return file_system_;
} }
#ifdef OS_WIN
std::unique_ptr<Env> NewCompositeEnv(std::shared_ptr<FileSystem> fs) {
return std::unique_ptr<Env>(new CompositeEnvWrapper(Env::Default(), fs));
}
#endif
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

18
env/env_posix.cc vendored
View File

@ -122,14 +122,9 @@ class PosixDynamicLibrary : public DynamicLibrary {
}; };
#endif // !ROCKSDB_NO_DYNAMIC_EXTENSION #endif // !ROCKSDB_NO_DYNAMIC_EXTENSION
class PosixEnv : public CompositeEnvWrapper { class PosixEnv : public CompositeEnv {
public: public:
// This constructor is for constructing non-default Envs, mainly by PosixEnv(const PosixEnv* default_env, const std::shared_ptr<FileSystem>& fs);
// NewCompositeEnv(). It allows new instances to share the same
// threadpool and other resources as the default Env, while allowing
// a non-default FileSystem implementation
PosixEnv(const PosixEnv* default_env, std::shared_ptr<FileSystem> fs);
~PosixEnv() override { ~PosixEnv() override {
if (this == Env::Default()) { if (this == Env::Default()) {
for (const auto tid : threads_to_join_) { for (const auto tid : threads_to_join_) {
@ -387,7 +382,7 @@ class PosixEnv : public CompositeEnvWrapper {
}; };
PosixEnv::PosixEnv() PosixEnv::PosixEnv()
: CompositeEnvWrapper(this, FileSystem::Default()), : CompositeEnv(FileSystem::Default()),
thread_pools_storage_(Priority::TOTAL), thread_pools_storage_(Priority::TOTAL),
allow_non_owner_access_storage_(true), allow_non_owner_access_storage_(true),
thread_pools_(thread_pools_storage_), thread_pools_(thread_pools_storage_),
@ -404,8 +399,9 @@ PosixEnv::PosixEnv()
thread_status_updater_ = CreateThreadStatusUpdater(); thread_status_updater_ = CreateThreadStatusUpdater();
} }
PosixEnv::PosixEnv(const PosixEnv* default_env, std::shared_ptr<FileSystem> fs) PosixEnv::PosixEnv(const PosixEnv* default_env,
: CompositeEnvWrapper(this, fs), const std::shared_ptr<FileSystem>& fs)
: CompositeEnv(fs),
thread_pools_(default_env->thread_pools_), thread_pools_(default_env->thread_pools_),
mu_(default_env->mu_), mu_(default_env->mu_),
threads_to_join_(default_env->threads_to_join_), threads_to_join_(default_env->threads_to_join_),
@ -508,7 +504,7 @@ Env* Env::Default() {
return &default_env; return &default_env;
} }
std::unique_ptr<Env> NewCompositeEnv(std::shared_ptr<FileSystem> fs) { std::unique_ptr<Env> NewCompositeEnv(const std::shared_ptr<FileSystem>& fs) {
PosixEnv* default_env = static_cast<PosixEnv*>(Env::Default()); PosixEnv* default_env = static_cast<PosixEnv*>(Env::Default());
return std::unique_ptr<Env>(new PosixEnv(default_env, fs)); return std::unique_ptr<Env>(new PosixEnv(default_env, fs));
} }

9
env/file_system.cc vendored
View File

@ -129,13 +129,4 @@ IOStatus ReadFileToString(FileSystem* fs, const std::string& fname,
return s; return s;
} }
#ifdef OS_WIN
std::shared_ptr<FileSystem> FileSystem::Default() {
static LegacyFileSystemWrapper default_fs(Env::Default());
static std::shared_ptr<LegacyFileSystemWrapper> default_fs_ptr(
&default_fs, [](LegacyFileSystemWrapper*) {});
return default_fs_ptr;
}
#endif
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

View File

@ -10,7 +10,6 @@
#include <thread> #include <thread>
#include <vector> #include <vector>
#include "env/composite_env_wrapper.h"
#include "file/file_util.h" #include "file/file_util.h"
#include "file/sst_file_manager_impl.h" #include "file/sst_file_manager_impl.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
@ -96,10 +95,8 @@ class DeleteSchedulerTest : public testing::Test {
// Tests in this file are for DeleteScheduler component and don't create any // Tests in this file are for DeleteScheduler component and don't create any
// DBs, so we need to set max_trash_db_ratio to 100% (instead of default // DBs, so we need to set max_trash_db_ratio to 100% (instead of default
// 25%) // 25%)
std::shared_ptr<FileSystem> sst_file_mgr_.reset(new SstFileManagerImpl(
fs(std::make_shared<LegacyFileSystemWrapper>(env_)); env_, env_->GetFileSystem(), nullptr, rate_bytes_per_sec_,
sst_file_mgr_.reset(
new SstFileManagerImpl(env_, fs, nullptr, rate_bytes_per_sec_,
/* max_trash_db_ratio= */ 1.1, 128 * 1024)); /* max_trash_db_ratio= */ 1.1, 128 * 1024));
delete_scheduler_ = sst_file_mgr_->delete_scheduler(); delete_scheduler_ = sst_file_mgr_->delete_scheduler();
sst_file_mgr_->SetStatisticsPtr(stats_); sst_file_mgr_->SetStatisticsPtr(stats_);

View File

@ -22,10 +22,23 @@ extern IOStatus CopyFile(FileSystem* fs, const std::string& source,
const std::string& destination, uint64_t size, const std::string& destination, uint64_t size,
bool use_fsync, bool use_fsync,
const std::shared_ptr<IOTracer>& io_tracer = nullptr); const std::shared_ptr<IOTracer>& io_tracer = nullptr);
inline IOStatus CopyFile(const std::shared_ptr<FileSystem>& fs,
const std::string& source,
const std::string& destination, uint64_t size,
bool use_fsync,
const std::shared_ptr<IOTracer>& io_tracer = nullptr) {
return CopyFile(fs.get(), source, destination, size, use_fsync, io_tracer);
}
extern IOStatus CreateFile(FileSystem* fs, const std::string& destination, extern IOStatus CreateFile(FileSystem* fs, const std::string& destination,
const std::string& contents, bool use_fsync); const std::string& contents, bool use_fsync);
inline IOStatus CreateFile(const std::shared_ptr<FileSystem>& fs,
const std::string& destination,
const std::string& contents, bool use_fsync) {
return CreateFile(fs.get(), destination, contents, use_fsync);
}
extern Status DeleteDBFile(const ImmutableDBOptions* db_options, extern Status DeleteDBFile(const ImmutableDBOptions* db_options,
const std::string& fname, const std::string& fname,
const std::string& path_to_sync, const bool force_bg, const std::string& path_to_sync, const bool force_bg,
@ -41,6 +54,19 @@ extern IOStatus GenerateOneFileChecksum(
size_t verify_checksums_readahead_size, bool allow_mmap_reads, size_t verify_checksums_readahead_size, bool allow_mmap_reads,
std::shared_ptr<IOTracer>& io_tracer, RateLimiter* rate_limiter = nullptr); std::shared_ptr<IOTracer>& io_tracer, RateLimiter* rate_limiter = nullptr);
inline IOStatus GenerateOneFileChecksum(
const std::shared_ptr<FileSystem>& fs, const std::string& file_path,
FileChecksumGenFactory* checksum_factory,
const std::string& requested_checksum_func_name, std::string* file_checksum,
std::string* file_checksum_func_name,
size_t verify_checksums_readahead_size, bool allow_mmap_reads,
std::shared_ptr<IOTracer>& io_tracer) {
return GenerateOneFileChecksum(
fs.get(), file_path, checksum_factory, requested_checksum_func_name,
file_checksum, file_checksum_func_name, verify_checksums_readahead_size,
allow_mmap_reads, io_tracer);
}
inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro, Env* env, inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro, Env* env,
IOOptions& opts) { IOOptions& opts) {
if (!env) { if (!env) {

View File

@ -75,7 +75,9 @@ std::string BuildKey(int num, std::string postfix = "") {
TEST_P(PrefetchTest, Basic) { TEST_P(PrefetchTest, Basic) {
// First param is if the mockFS support_prefetch or not // First param is if the mockFS support_prefetch or not
bool support_prefetch = std::get<0>(GetParam()); bool support_prefetch =
std::get<0>(GetParam()) &&
test::IsPrefetchSupported(env_->GetFileSystem(), dbname_);
// Second param is if directIO is enabled or not // Second param is if directIO is enabled or not
bool use_direct_io = std::get<1>(GetParam()); bool use_direct_io = std::get<1>(GetParam());

View File

@ -9,7 +9,6 @@
#include <vector> #include <vector>
#include "db/db_impl/db_impl.h" #include "db/db_impl/db_impl.h"
#include "env/composite_env_wrapper.h"
#include "port/port.h" #include "port/port.h"
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "rocksdb/sst_file_manager.h" #include "rocksdb/sst_file_manager.h"
@ -485,13 +484,7 @@ SstFileManager* NewSstFileManager(Env* env, std::shared_ptr<Logger> info_log,
bool delete_existing_trash, Status* status, bool delete_existing_trash, Status* status,
double max_trash_db_ratio, double max_trash_db_ratio,
uint64_t bytes_max_delete_chunk) { uint64_t bytes_max_delete_chunk) {
std::shared_ptr<FileSystem> fs; const auto& fs = env->GetFileSystem();
if (env == Env::Default()) {
fs = FileSystem::Default();
} else {
fs.reset(new LegacyFileSystemWrapper(env));
}
return NewSstFileManager(env, fs, info_log, trash_dir, rate_bytes_per_sec, return NewSstFileManager(env, fs, info_log, trash_dir, rate_bytes_per_sec,
delete_existing_trash, status, max_trash_db_ratio, delete_existing_trash, status, max_trash_db_ratio,

View File

@ -30,6 +30,7 @@
// Windows API macro interference // Windows API macro interference
#undef DeleteFile #undef DeleteFile
#undef GetCurrentTime #undef GetCurrentTime
#undef LoadLibrary
#endif #endif
#if defined(__GNUC__) || defined(__clang__) #if defined(__GNUC__) || defined(__clang__)
@ -1663,6 +1664,6 @@ Env* NewTimedEnv(Env* base_env);
Status NewEnvLogger(const std::string& fname, Env* env, Status NewEnvLogger(const std::string& fname, Env* env,
std::shared_ptr<Logger>* result); std::shared_ptr<Logger>* result);
std::unique_ptr<Env> NewCompositeEnv(std::shared_ptr<FileSystem> fs); std::unique_ptr<Env> NewCompositeEnv(const std::shared_ptr<FileSystem>& fs);
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

View File

@ -366,6 +366,10 @@ class FileSystem {
return IOStatus::OK(); return IOStatus::OK();
} }
// This seems to clash with a macro on Windows, so #undef it here
#ifdef DeleteFile
#undef DeleteFile
#endif
// Delete the named file. // Delete the named file.
virtual IOStatus DeleteFile(const std::string& fname, virtual IOStatus DeleteFile(const std::string& fname,
const IOOptions& options, const IOOptions& options,
@ -1048,7 +1052,8 @@ class FSDirectory {
class FileSystemWrapper : public FileSystem { class FileSystemWrapper : public FileSystem {
public: public:
// Initialize an EnvWrapper that delegates all calls to *t // Initialize an EnvWrapper that delegates all calls to *t
explicit FileSystemWrapper(std::shared_ptr<FileSystem> t) : target_(t) {} explicit FileSystemWrapper(const std::shared_ptr<FileSystem>& t)
: target_(t) {}
~FileSystemWrapper() override {} ~FileSystemWrapper() override {}
const char* Name() const override { return target_->Name(); } const char* Name() const override { return target_->Name(); }

View File

@ -11,8 +11,8 @@
#include <mutex> #include <mutex>
#include <rocksdb/env.h>
#include "port/win/env_win.h" #include "port/win/env_win.h"
#include "rocksdb/env.h"
#include "test_util/sync_point.h" #include "test_util/sync_point.h"
#include "util/compression_context_cache.h" #include "util/compression_context_cache.h"
#include "util/thread_local.h" #include "util/thread_local.h"

File diff suppressed because it is too large Load Diff

View File

@ -15,30 +15,29 @@
// multiple threads without any external synchronization. // multiple threads without any external synchronization.
#pragma once #pragma once
#include "port/win/win_thread.h"
#include <rocksdb/env.h>
#include "util/threadpool_imp.h"
#include <stdint.h> #include <stdint.h>
#include <windows.h> #include <windows.h>
#include <mutex> #include <mutex>
#include <vector>
#include <string> #include <string>
#include <vector>
#include "env/composite_env_wrapper.h"
#include "port/win/win_thread.h"
#include "rocksdb/env.h"
#include "rocksdb/file_system.h"
#include "util/threadpool_imp.h"
#undef GetCurrentTime #undef GetCurrentTime
#undef DeleteFile #undef DeleteFile
#undef GetTickCount #undef LoadLibrary
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace port { namespace port {
// Currently not designed for inheritance but rather a replacement // Currently not designed for inheritance but rather a replacement
class WinEnvThreads { class WinEnvThreads {
public: public:
explicit WinEnvThreads(Env* hosted_env); explicit WinEnvThreads(Env* hosted_env);
~WinEnvThreads(); ~WinEnvThreads();
@ -46,12 +45,12 @@ public:
WinEnvThreads(const WinEnvThreads&) = delete; WinEnvThreads(const WinEnvThreads&) = delete;
WinEnvThreads& operator=(const WinEnvThreads&) = delete; WinEnvThreads& operator=(const WinEnvThreads&) = delete;
void Schedule(void(*function)(void*), void* arg, Env::Priority pri, void Schedule(void (*function)(void*), void* arg, Env::Priority pri,
void* tag, void(*unschedFunction)(void* arg)); void* tag, void (*unschedFunction)(void* arg));
int UnSchedule(void* arg, Env::Priority pri); int UnSchedule(void* arg, Env::Priority pri);
void StartThread(void(*function)(void* arg), void* arg); void StartThread(void (*function)(void* arg), void* arg);
void WaitForJoin(); void WaitForJoin();
@ -61,235 +60,198 @@ public:
uint64_t GetThreadID() const; uint64_t GetThreadID() const;
void SleepForMicroseconds(int micros);
// Allow increasing the number of worker threads. // Allow increasing the number of worker threads.
void SetBackgroundThreads(int num, Env::Priority pri); void SetBackgroundThreads(int num, Env::Priority pri);
int GetBackgroundThreads(Env::Priority pri); int GetBackgroundThreads(Env::Priority pri);
void IncBackgroundThreadsIfNeeded(int num, Env::Priority pri); void IncBackgroundThreadsIfNeeded(int num, Env::Priority pri);
private: private:
Env* hosted_env_; Env* hosted_env_;
mutable std::mutex mu_; mutable std::mutex mu_;
std::vector<ThreadPoolImpl> thread_pools_; std::vector<ThreadPoolImpl> thread_pools_;
std::vector<WindowsThread> threads_to_join_; std::vector<WindowsThread> threads_to_join_;
}; };
// Designed for inheritance so can be re-used class WinClock {
// but certain parts replaced public:
class WinEnvIO { static const std::shared_ptr<WinClock>& Default();
public: WinClock();
explicit WinEnvIO(Env* hosted_env); virtual ~WinClock() {}
virtual ~WinEnvIO();
virtual Status DeleteFile(const std::string& fname);
Status Truncate(const std::string& fname, size_t size);
virtual Status GetCurrentTime(int64_t* unix_time);
virtual Status NewSequentialFile(const std::string& fname,
std::unique_ptr<SequentialFile>* result,
const EnvOptions& options);
// Helper for NewWritable and ReopenWritableFile
virtual Status OpenWritableFile(const std::string& fname,
std::unique_ptr<WritableFile>* result,
const EnvOptions& options,
bool reopen);
virtual Status NewRandomAccessFile(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result,
const EnvOptions& options);
// The returned file will only be accessed by one thread at a time.
virtual Status NewRandomRWFile(const std::string& fname,
std::unique_ptr<RandomRWFile>* result,
const EnvOptions& options);
virtual Status NewMemoryMappedFileBuffer(
const std::string& fname,
std::unique_ptr<MemoryMappedFileBuffer>* result);
virtual Status NewDirectory(const std::string& name,
std::unique_ptr<Directory>* result);
virtual Status FileExists(const std::string& fname);
virtual Status GetChildren(const std::string& dir,
std::vector<std::string>* result);
virtual Status CreateDir(const std::string& name);
virtual Status CreateDirIfMissing(const std::string& name);
virtual Status DeleteDir(const std::string& name);
virtual Status GetFileSize(const std::string& fname, uint64_t* size);
static uint64_t FileTimeToUnixTime(const FILETIME& ftTime);
virtual Status GetFileModificationTime(const std::string& fname,
uint64_t* file_mtime);
virtual Status RenameFile(const std::string& src, const std::string& target);
virtual Status LinkFile(const std::string& src, const std::string& target);
virtual Status NumFileLinks(const std::string& /*fname*/,
uint64_t* /*count*/);
virtual Status AreFilesSame(const std::string& first,
const std::string& second, bool* res);
virtual Status LockFile(const std::string& lockFname, FileLock** lock);
virtual Status UnlockFile(FileLock* lock);
virtual Status GetTestDirectory(std::string* result);
virtual Status NewLogger(const std::string& fname,
std::shared_ptr<Logger>* result);
virtual Status IsDirectory(const std::string& path, bool* is_dir);
virtual uint64_t NowMicros(); virtual uint64_t NowMicros();
virtual uint64_t NowNanos(); virtual uint64_t NowNanos();
virtual Status GetHostName(char* name, uint64_t len); virtual void SleepForMicroseconds(int micros);
virtual Status GetAbsolutePath(const std::string& db_path, virtual Status GetCurrentTime(int64_t* unix_time);
std::string* output_path); // Converts seconds-since-Jan-01-1970 to a printable string
virtual std::string TimeToString(uint64_t time);
// This seems to clash with a macro on Windows, so #undef it here
#undef GetFreeSpace
// Get the amount of free disk space
virtual Status GetFreeSpace(const std::string& path, uint64_t* diskfree);
virtual std::string TimeToString(uint64_t secondsSince1970);
virtual EnvOptions OptimizeForLogWrite(const EnvOptions& env_options,
const DBOptions& db_options) const;
virtual EnvOptions OptimizeForManifestWrite(
const EnvOptions& env_options) const;
virtual EnvOptions OptimizeForManifestRead(
const EnvOptions& env_options) const;
size_t GetPageSize() const { return page_size_; }
size_t GetAllocationGranularity() const { return allocation_granularity_; }
uint64_t GetPerfCounterFrequency() const { return perf_counter_frequency_; } uint64_t GetPerfCounterFrequency() const { return perf_counter_frequency_; }
static size_t GetSectorSize(const std::string& fname); private:
typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME);
private:
// Returns true iff the named directory exists and is a directory.
virtual bool DirExists(const std::string& dname);
typedef VOID(WINAPI * FnGetSystemTimePreciseAsFileTime)(LPFILETIME);
Env* hosted_env_;
size_t page_size_;
size_t allocation_granularity_;
uint64_t perf_counter_frequency_; uint64_t perf_counter_frequency_;
uint64_t nano_seconds_per_period_; uint64_t nano_seconds_per_period_;
FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_; FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_;
}; };
class WinEnv : public Env { class WinFileSystem : public FileSystem {
public: public:
WinEnv(); static const std::shared_ptr<WinFileSystem>& Default();
WinFileSystem(const std::shared_ptr<WinClock>& clock);
~WinFileSystem() {}
const char* Name() const { return "WinFS"; }
static size_t GetSectorSize(const std::string& fname);
size_t GetPageSize() const { return page_size_; }
size_t GetAllocationGranularity() const { return allocation_granularity_; }
~WinEnv(); IOStatus DeleteFile(const std::string& fname, const IOOptions& options,
IODebugContext* dbg) override;
Status DeleteFile(const std::string& fname) override; // Truncate the named file to the specified size.
IOStatus Truncate(const std::string& /*fname*/, size_t /*size*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override;
IOStatus NewSequentialFile(const std::string& fname,
const FileOptions& file_opts,
std::unique_ptr<FSSequentialFile>* result,
IODebugContext* dbg) override;
Status Truncate(const std::string& fname, size_t size) override; IOStatus NewRandomAccessFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSRandomAccessFile>* result,
IODebugContext* /*dbg*/) override;
IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts,
std::unique_ptr<FSWritableFile>* r,
IODebugContext* dbg) override;
IOStatus ReopenWritableFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
IODebugContext* dbg) override;
Status GetCurrentTime(int64_t* unix_time) override; IOStatus NewRandomRWFile(const std::string& fname,
const FileOptions& file_opts,
Status NewSequentialFile(const std::string& fname, std::unique_ptr<FSRandomRWFile>* result,
std::unique_ptr<SequentialFile>* result, IODebugContext* dbg) override;
const EnvOptions& options) override; IOStatus NewMemoryMappedFileBuffer(
Status NewRandomAccessFile(const std::string& fname,
std::unique_ptr<RandomAccessFile>* result,
const EnvOptions& options) override;
Status NewWritableFile(const std::string& fname,
std::unique_ptr<WritableFile>* result,
const EnvOptions& options) override;
// Create an object that writes to a new file with the specified
// name. Deletes any existing file with the same name and creates a
// new file. On success, stores a pointer to the new file in
// *result and returns OK. On failure stores nullptr in *result and
// returns non-OK.
//
// The returned file will only be accessed by one thread at a time.
Status ReopenWritableFile(const std::string& fname,
std::unique_ptr<WritableFile>* result,
const EnvOptions& options) override;
// The returned file will only be accessed by one thread at a time.
Status NewRandomRWFile(const std::string& fname,
std::unique_ptr<RandomRWFile>* result,
const EnvOptions& options) override;
Status NewMemoryMappedFileBuffer(
const std::string& fname, const std::string& fname,
std::unique_ptr<MemoryMappedFileBuffer>* result) override; std::unique_ptr<MemoryMappedFileBuffer>* result) override;
Status NewDirectory(const std::string& name, IOStatus NewDirectory(const std::string& name, const IOOptions& io_opts,
std::unique_ptr<Directory>* result) override; std::unique_ptr<FSDirectory>* result,
IODebugContext* dbg) override;
IOStatus FileExists(const std::string& f, const IOOptions& io_opts,
IODebugContext* dbg) override;
IOStatus GetChildren(const std::string& dir, const IOOptions& io_opts,
std::vector<std::string>* r,
IODebugContext* dbg) override;
IOStatus CreateDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
Status FileExists(const std::string& fname) override; // Creates directory if missing. Return Ok if it exists, or successful in
// Creating.
IOStatus CreateDirIfMissing(const std::string& dirname,
const IOOptions& options,
IODebugContext* dbg) override;
Status GetChildren(const std::string& dir, // Delete the specified directory.
std::vector<std::string>* result) override; IOStatus DeleteDir(const std::string& dirname, const IOOptions& options,
IODebugContext* dbg) override;
// Store the size of fname in *file_size.
IOStatus GetFileSize(const std::string& fname, const IOOptions& options,
uint64_t* file_size, IODebugContext* dbg) override;
// Store the last modification time of fname in *file_mtime.
IOStatus GetFileModificationTime(const std::string& fname,
const IOOptions& options,
uint64_t* file_mtime,
IODebugContext* dbg) override;
// Rename file src to target.
IOStatus RenameFile(const std::string& src, const std::string& target,
const IOOptions& options, IODebugContext* dbg) override;
Status CreateDir(const std::string& name) override; // Hard Link file src to target.
IOStatus LinkFile(const std::string& /*src*/, const std::string& /*target*/,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) override;
IOStatus NumFileLinks(const std::string& /*fname*/,
const IOOptions& /*options*/, uint64_t* /*count*/,
IODebugContext* /*dbg*/) override;
IOStatus AreFilesSame(const std::string& /*first*/,
const std::string& /*second*/,
const IOOptions& /*options*/, bool* /*res*/,
IODebugContext* /*dbg*/) override;
IOStatus LockFile(const std::string& fname, const IOOptions& options,
FileLock** lock, IODebugContext* dbg) override;
IOStatus UnlockFile(FileLock* lock, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus GetTestDirectory(const IOOptions& options, std::string* path,
IODebugContext* dbg) override;
Status CreateDirIfMissing(const std::string& name) override; // Create and returns a default logger (an instance of EnvLogger) for storing
// informational messages. Derived classes can overide to provide custom
// logger.
IOStatus NewLogger(const std::string& fname, const IOOptions& io_opts,
std::shared_ptr<Logger>* result,
IODebugContext* dbg) override;
// Get full directory name for this db.
IOStatus GetAbsolutePath(const std::string& db_path, const IOOptions& options,
std::string* output_path,
IODebugContext* dbg) override;
IOStatus IsDirectory(const std::string& /*path*/, const IOOptions& options,
bool* is_dir, IODebugContext* /*dgb*/) override;
// This seems to clash with a macro on Windows, so #undef it here
#undef GetFreeSpace
IOStatus GetFreeSpace(const std::string& /*path*/,
const IOOptions& /*options*/, uint64_t* /*diskfree*/,
IODebugContext* /*dbg*/) override;
FileOptions OptimizeForLogWrite(const FileOptions& file_options,
const DBOptions& db_options) const override;
FileOptions OptimizeForManifestRead(
const FileOptions& file_options) const override;
FileOptions OptimizeForManifestWrite(
const FileOptions& file_options) const override;
Status DeleteDir(const std::string& name) override; protected:
static uint64_t FileTimeToUnixTime(const FILETIME& ftTime);
// Returns true iff the named directory exists and is a directory.
Status GetFileSize(const std::string& fname, virtual bool DirExists(const std::string& dname);
uint64_t* size) override; // Helper for NewWritable and ReopenWritableFile
virtual IOStatus OpenWritableFile(const std::string& fname,
const FileOptions& options,
std::unique_ptr<FSWritableFile>* result,
bool reopen);
Status GetFileModificationTime(const std::string& fname, private:
uint64_t* file_mtime) override; std::shared_ptr<WinClock> clock_;
size_t page_size_;
size_t allocation_granularity_;
};
Status RenameFile(const std::string& src, // Designed for inheritance so can be re-used
const std::string& target) override; // but certain parts replaced
class WinEnvIO {
public:
explicit WinEnvIO(Env* hosted_env);
Status LinkFile(const std::string& src, virtual ~WinEnvIO();
const std::string& target) override;
Status NumFileLinks(const std::string& fname, uint64_t* count) override; virtual Status GetHostName(char* name, uint64_t len);
Status AreFilesSame(const std::string& first, private:
const std::string& second, bool* res) override; Env* hosted_env_;
};
Status LockFile(const std::string& lockFname, FileLock** lock) override; class WinEnv : public CompositeEnv {
public:
WinEnv();
Status UnlockFile(FileLock* lock) override; ~WinEnv();
Status GetCurrentTime(int64_t* unix_time) override;
Status GetTestDirectory(std::string* result) override;
Status NewLogger(const std::string& fname,
std::shared_ptr<Logger>* result) override;
Status IsDirectory(const std::string& path, bool* is_dir) override;
uint64_t NowMicros() override; uint64_t NowMicros() override;
@ -297,19 +259,16 @@ public:
Status GetHostName(char* name, uint64_t len) override; Status GetHostName(char* name, uint64_t len) override;
Status GetAbsolutePath(const std::string& db_path,
std::string* output_path) override;
std::string TimeToString(uint64_t secondsSince1970) override; std::string TimeToString(uint64_t secondsSince1970) override;
Status GetThreadList(std::vector<ThreadStatus>* thread_list) override; Status GetThreadList(std::vector<ThreadStatus>* thread_list) override;
void Schedule(void(*function)(void*), void* arg, Env::Priority pri, void Schedule(void (*function)(void*), void* arg, Env::Priority pri,
void* tag, void(*unschedFunction)(void* arg)) override; void* tag, void (*unschedFunction)(void* arg)) override;
int UnSchedule(void* arg, Env::Priority pri) override; int UnSchedule(void* arg, Env::Priority pri) override;
void StartThread(void(*function)(void* arg), void* arg) override; void StartThread(void (*function)(void* arg), void* arg) override;
void WaitForJoin() override; void WaitForJoin() override;
@ -317,12 +276,6 @@ public:
uint64_t GetThreadID() const override; uint64_t GetThreadID() const override;
// This seems to clash with a macro on Windows, so #undef it here
#undef GetFreeSpace
// Get the amount of free disk space
Status GetFreeSpace(const std::string& path, uint64_t* diskfree) override;
void SleepForMicroseconds(int micros) override; void SleepForMicroseconds(int micros) override;
// Allow increasing the number of worker threads. // Allow increasing the number of worker threads.
@ -331,18 +284,8 @@ public:
void IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) override; void IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) override;
EnvOptions OptimizeForManifestRead( private:
const EnvOptions& env_options) const override; std::shared_ptr<WinClock> clock_;
EnvOptions OptimizeForLogWrite(const EnvOptions& env_options,
const DBOptions& db_options) const override;
EnvOptions OptimizeForManifestWrite(
const EnvOptions& env_options) const override;
private:
WinEnvIO winenv_io_; WinEnvIO winenv_io_;
WinEnvThreads winenv_threads_; WinEnvThreads winenv_threads_;
}; };

View File

@ -20,28 +20,24 @@ namespace ROCKSDB_NAMESPACE {
namespace port { namespace port {
/* /*
* DirectIOHelper * DirectIOHelper
*/ */
namespace { namespace {
const size_t kSectorSize = 512; const size_t kSectorSize = 512;
inline inline bool IsPowerOfTwo(const size_t alignment) {
bool IsPowerOfTwo(const size_t alignment) {
return ((alignment) & (alignment - 1)) == 0; return ((alignment) & (alignment - 1)) == 0;
} }
inline inline bool IsSectorAligned(const size_t off) {
bool IsSectorAligned(const size_t off) {
return (off & (kSectorSize - 1)) == 0; return (off & (kSectorSize - 1)) == 0;
} }
inline inline bool IsAligned(size_t alignment, const void* ptr) {
bool IsAligned(size_t alignment, const void* ptr) {
return ((uintptr_t(ptr)) & (alignment - 1)) == 0; return ((uintptr_t(ptr)) & (alignment - 1)) == 0;
} }
} } // namespace
std::string GetWindowsErrSz(DWORD err) { std::string GetWindowsErrSz(DWORD err) {
LPSTR lpMsgBuf; LPSTR lpMsgBuf;
@ -69,21 +65,20 @@ std::string GetWindowsErrSz(DWORD err) {
// Because all the reads/writes happen by the specified offset, the caller in // Because all the reads/writes happen by the specified offset, the caller in
// theory should not // theory should not
// rely on the current file offset. // rely on the current file offset.
Status pwrite(const WinFileData* file_data, const Slice& data, IOStatus pwrite(const WinFileData* file_data, const Slice& data,
uint64_t offset, size_t& bytes_written) { uint64_t offset, size_t& bytes_written) {
IOStatus s;
Status s;
bytes_written = 0; bytes_written = 0;
size_t num_bytes = data.size(); size_t num_bytes = data.size();
if (num_bytes > std::numeric_limits<DWORD>::max()) { if (num_bytes > std::numeric_limits<DWORD>::max()) {
// May happen in 64-bit builds where size_t is 64-bits but // May happen in 64-bit builds where size_t is 64-bits but
// long is still 32-bit, but that's the API here at the moment // long is still 32-bit, but that's the API here at the moment
return Status::InvalidArgument("num_bytes is too large for a single write: " + return IOStatus::InvalidArgument(
file_data->GetName()); "num_bytes is too large for a single write: " + file_data->GetName());
} }
OVERLAPPED overlapped = { 0 }; OVERLAPPED overlapped = {0};
ULARGE_INTEGER offsetUnion; ULARGE_INTEGER offsetUnion;
offsetUnion.QuadPart = offset; offsetUnion.QuadPart = offset;
@ -92,8 +87,9 @@ Status pwrite(const WinFileData* file_data, const Slice& data,
DWORD bytesWritten = 0; DWORD bytesWritten = 0;
if (FALSE == WriteFile(file_data->GetFileHandle(), data.data(), static_cast<DWORD>(num_bytes), if (FALSE == WriteFile(file_data->GetFileHandle(), data.data(),
&bytesWritten, &overlapped)) { static_cast<DWORD>(num_bytes), &bytesWritten,
&overlapped)) {
auto lastError = GetLastError(); auto lastError = GetLastError();
s = IOErrorFromWindowsError("WriteFile failed: " + file_data->GetName(), s = IOErrorFromWindowsError("WriteFile failed: " + file_data->GetName(),
lastError); lastError);
@ -105,18 +101,17 @@ Status pwrite(const WinFileData* file_data, const Slice& data,
} }
// See comments for pwrite above // See comments for pwrite above
Status pread(const WinFileData* file_data, char* src, size_t num_bytes, IOStatus pread(const WinFileData* file_data, char* src, size_t num_bytes,
uint64_t offset, size_t& bytes_read) { uint64_t offset, size_t& bytes_read) {
IOStatus s;
Status s;
bytes_read = 0; bytes_read = 0;
if (num_bytes > std::numeric_limits<DWORD>::max()) { if (num_bytes > std::numeric_limits<DWORD>::max()) {
return Status::InvalidArgument("num_bytes is too large for a single read: " + return IOStatus::InvalidArgument(
file_data->GetName()); "num_bytes is too large for a single read: " + file_data->GetName());
} }
OVERLAPPED overlapped = { 0 }; OVERLAPPED overlapped = {0};
ULARGE_INTEGER offsetUnion; ULARGE_INTEGER offsetUnion;
offsetUnion.QuadPart = offset; offsetUnion.QuadPart = offset;
@ -125,8 +120,9 @@ Status pread(const WinFileData* file_data, char* src, size_t num_bytes,
DWORD bytesRead = 0; DWORD bytesRead = 0;
if (FALSE == ReadFile(file_data->GetFileHandle(), src, static_cast<DWORD>(num_bytes), if (FALSE == ReadFile(file_data->GetFileHandle(), src,
&bytesRead, &overlapped)) { static_cast<DWORD>(num_bytes), &bytesRead,
&overlapped)) {
auto lastError = GetLastError(); auto lastError = GetLastError();
// EOF is OK with zero bytes read // EOF is OK with zero bytes read
if (lastError != ERROR_HANDLE_EOF) { if (lastError != ERROR_HANDLE_EOF) {
@ -143,9 +139,9 @@ Status pread(const WinFileData* file_data, char* src, size_t num_bytes,
// SetFileInformationByHandle() is capable of fast pre-allocates. // SetFileInformationByHandle() is capable of fast pre-allocates.
// However, this does not change the file end position unless the file is // However, this does not change the file end position unless the file is
// truncated and the pre-allocated space is not considered filled with zeros. // truncated and the pre-allocated space is not considered filled with zeros.
Status fallocate(const std::string& filename, HANDLE hFile, IOStatus fallocate(const std::string& filename, HANDLE hFile,
uint64_t to_size) { uint64_t to_size) {
Status status; IOStatus status;
FILE_ALLOCATION_INFO alloc_info; FILE_ALLOCATION_INFO alloc_info;
alloc_info.AllocationSize.QuadPart = to_size; alloc_info.AllocationSize.QuadPart = to_size;
@ -160,9 +156,8 @@ Status fallocate(const std::string& filename, HANDLE hFile,
return status; return status;
} }
Status ftruncate(const std::string& filename, HANDLE hFile, IOStatus ftruncate(const std::string& filename, HANDLE hFile, uint64_t toSize) {
uint64_t toSize) { IOStatus status;
Status status;
FILE_END_OF_FILE_INFO end_of_file; FILE_END_OF_FILE_INFO end_of_file;
end_of_file.EndOfFile.QuadPart = toSize; end_of_file.EndOfFile.QuadPart = toSize;
@ -212,9 +207,11 @@ WinMmapReadableFile::~WinMmapReadableFile() {
assert(ret); assert(ret);
} }
Status WinMmapReadableFile::Read(uint64_t offset, size_t n, Slice* result, IOStatus WinMmapReadableFile::Read(uint64_t offset, size_t n,
char* scratch) const { const IOOptions& /*options*/, Slice* result,
Status s; char* scratch,
IODebugContext* /*dbg*/) const {
IOStatus s;
if (offset > length_) { if (offset > length_) {
*result = Slice(); *result = Slice();
@ -222,13 +219,12 @@ Status WinMmapReadableFile::Read(uint64_t offset, size_t n, Slice* result,
} else if (offset + n > length_) { } else if (offset + n > length_) {
n = length_ - static_cast<size_t>(offset); n = length_ - static_cast<size_t>(offset);
} }
*result = *result = Slice(reinterpret_cast<const char*>(mapped_region_) + offset, n);
Slice(reinterpret_cast<const char*>(mapped_region_)+offset, n);
return s; return s;
} }
Status WinMmapReadableFile::InvalidateCache(size_t offset, size_t length) { IOStatus WinMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
return Status::OK(); return IOStatus::OK();
} }
size_t WinMmapReadableFile::GetUniqueId(char* id, size_t max_size) const { size_t WinMmapReadableFile::GetUniqueId(char* id, size_t max_size) const {
@ -238,15 +234,14 @@ size_t WinMmapReadableFile::GetUniqueId(char* id, size_t max_size) const {
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
/// WinMmapFile /// WinMmapFile
// Can only truncate or reserve to a sector size aligned if // Can only truncate or reserve to a sector size aligned if
// used on files that are opened with Unbuffered I/O // used on files that are opened with Unbuffered I/O
Status WinMmapFile::TruncateFile(uint64_t toSize) { IOStatus WinMmapFile::TruncateFile(uint64_t toSize) {
return ftruncate(filename_, hFile_, toSize); return ftruncate(filename_, hFile_, toSize);
} }
Status WinMmapFile::UnmapCurrentRegion() { IOStatus WinMmapFile::UnmapCurrentRegion() {
Status status; IOStatus status;
if (mapped_begin_ != nullptr) { if (mapped_begin_ != nullptr) {
if (!::UnmapViewOfFile(mapped_begin_)) { if (!::UnmapViewOfFile(mapped_begin_)) {
@ -271,16 +266,16 @@ Status WinMmapFile::UnmapCurrentRegion() {
return status; return status;
} }
Status WinMmapFile::MapNewRegion() { IOStatus WinMmapFile::MapNewRegion(const IOOptions& options,
IODebugContext* dbg) {
Status status; IOStatus status;
assert(mapped_begin_ == nullptr); assert(mapped_begin_ == nullptr);
size_t minDiskSize = static_cast<size_t>(file_offset_) + view_size_; size_t minDiskSize = static_cast<size_t>(file_offset_) + view_size_;
if (minDiskSize > reserved_size_) { if (minDiskSize > reserved_size_) {
status = Allocate(file_offset_, view_size_); status = Allocate(file_offset_, view_size_, options, dbg);
if (!status.ok()) { if (!status.ok()) {
return status; return status;
} }
@ -288,7 +283,6 @@ Status WinMmapFile::MapNewRegion() {
// Need to remap // Need to remap
if (hMap_ == NULL || reserved_size_ > mapping_size_) { if (hMap_ == NULL || reserved_size_ > mapping_size_) {
if (hMap_ != NULL) { if (hMap_ != NULL) {
// Unmap the previous one // Unmap the previous one
BOOL ret __attribute__((__unused__)); BOOL ret __attribute__((__unused__));
@ -339,15 +333,15 @@ Status WinMmapFile::MapNewRegion() {
return status; return status;
} }
Status WinMmapFile::PreallocateInternal(uint64_t spaceToReserve) { IOStatus WinMmapFile::PreallocateInternal(uint64_t spaceToReserve) {
return fallocate(filename_, hFile_, spaceToReserve); return fallocate(filename_, hFile_, spaceToReserve);
} }
WinMmapFile::WinMmapFile(const std::string& fname, HANDLE hFile, WinMmapFile::WinMmapFile(const std::string& fname, HANDLE hFile,
size_t page_size, size_t allocation_granularity, size_t page_size, size_t allocation_granularity,
const EnvOptions& options) const FileOptions& options)
: WinFileData(fname, hFile, false), : WinFileData(fname, hFile, false),
WritableFile(options), FSWritableFile(options),
hMap_(NULL), hMap_(NULL),
page_size_(page_size), page_size_(page_size),
allocation_granularity_(allocation_granularity), allocation_granularity_(allocation_granularity),
@ -373,17 +367,19 @@ WinMmapFile::WinMmapFile(const std::string& fname, HANDLE hFile,
// View size must be both the multiple of allocation_granularity AND the // View size must be both the multiple of allocation_granularity AND the
// page size and the granularity is usually a multiple of a page size. // page size and the granularity is usually a multiple of a page size.
const size_t viewSize = 32 * 1024; // 32Kb similar to the Windows File Cache in buffered mode const size_t viewSize =
32 * 1024; // 32Kb similar to the Windows File Cache in buffered mode
view_size_ = Roundup(viewSize, allocation_granularity_); view_size_ = Roundup(viewSize, allocation_granularity_);
} }
WinMmapFile::~WinMmapFile() { WinMmapFile::~WinMmapFile() {
if (hFile_) { if (hFile_) {
this->Close(); this->Close(IOOptions(), nullptr);
} }
} }
Status WinMmapFile::Append(const Slice& data) { IOStatus WinMmapFile::Append(const Slice& data, const IOOptions& options,
IODebugContext* dbg) {
const char* src = data.data(); const char* src = data.data();
size_t left = data.size(); size_t left = data.size();
@ -392,9 +388,9 @@ Status WinMmapFile::Append(const Slice& data) {
size_t avail = mapped_end_ - dst_; size_t avail = mapped_end_ - dst_;
if (avail == 0) { if (avail == 0) {
Status s = UnmapCurrentRegion(); IOStatus s = UnmapCurrentRegion();
if (s.ok()) { if (s.ok()) {
s = MapNewRegion(); s = MapNewRegion(options, dbg);
} }
if (!s.ok()) { if (!s.ok()) {
@ -416,30 +412,31 @@ Status WinMmapFile::Append(const Slice& data) {
memset(dst_, 0, bytesToPad); memset(dst_, 0, bytesToPad);
} }
return Status::OK(); return IOStatus::OK();
} }
// Means Close() will properly take care of truncate // Means Close() will properly take care of truncate
// and it does not need any additional information // and it does not need any additional information
Status WinMmapFile::Truncate(uint64_t size) { IOStatus WinMmapFile::Truncate(uint64_t size, const IOOptions& /*options*/,
return Status::OK(); IODebugContext* /*dbg*/) {
return IOStatus::OK();
} }
Status WinMmapFile::Close() { IOStatus WinMmapFile::Close(const IOOptions& options, IODebugContext* dbg) {
Status s; IOStatus s;
assert(NULL != hFile_); assert(NULL != hFile_);
// We truncate to the precise size so no // We truncate to the precise size so no
// uninitialized data at the end. SetEndOfFile // uninitialized data at the end. SetEndOfFile
// which we use does not write zeros and it is good. // which we use does not write zeros and it is good.
uint64_t targetSize = GetFileSize(); uint64_t targetSize = GetFileSize(options, dbg);
if (mapped_begin_ != nullptr) { if (mapped_begin_ != nullptr) {
// Sync before unmapping to make sure everything // Sync before unmapping to make sure everything
// is on disk and there is not a lazy writing // is on disk and there is not a lazy writing
// so we are deterministic with the tests // so we are deterministic with the tests
Sync(); Sync(options, dbg);
s = UnmapCurrentRegion(); s = UnmapCurrentRegion();
} }
@ -455,7 +452,6 @@ Status WinMmapFile::Close() {
} }
if (hFile_ != NULL) { if (hFile_ != NULL) {
TruncateFile(targetSize); TruncateFile(targetSize);
BOOL ret = ::CloseHandle(hFile_); BOOL ret = ::CloseHandle(hFile_);
@ -471,11 +467,15 @@ Status WinMmapFile::Close() {
return s; return s;
} }
Status WinMmapFile::Flush() { return Status::OK(); } IOStatus WinMmapFile::Flush(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return IOStatus::OK();
}
// Flush only data // Flush only data
Status WinMmapFile::Sync() { IOStatus WinMmapFile::Sync(const IOOptions& /*options*/,
Status s; IODebugContext* /*dbg*/) {
IOStatus s;
// Some writes occurred since last sync // Some writes occurred since last sync
if (dst_ > last_sync_) { if (dst_ > last_sync_) {
@ -503,10 +503,10 @@ Status WinMmapFile::Sync() {
} }
/** /**
* Flush data as well as metadata to stable storage. * Flush data as well as metadata to stable storage.
*/ */
Status WinMmapFile::Fsync() { IOStatus WinMmapFile::Fsync(const IOOptions& options, IODebugContext* dbg) {
Status s = Sync(); IOStatus s = Sync(options, dbg);
// Flush metadata // Flush metadata
if (s.ok() && pending_sync_) { if (s.ok() && pending_sync_) {
@ -521,27 +521,31 @@ Status WinMmapFile::Fsync() {
} }
/** /**
* Get the size of valid data in the file. This will not match the * Get the size of valid data in the file. This will not match the
* size that is returned from the filesystem because we use mmap * size that is returned from the filesystem because we use mmap
* to extend file by map_size every time. * to extend file by map_size every time.
*/ */
uint64_t WinMmapFile::GetFileSize() { uint64_t WinMmapFile::GetFileSize(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
size_t used = dst_ - mapped_begin_; size_t used = dst_ - mapped_begin_;
return file_offset_ + used; return file_offset_ + used;
} }
Status WinMmapFile::InvalidateCache(size_t offset, size_t length) { IOStatus WinMmapFile::InvalidateCache(size_t offset, size_t length) {
return Status::OK(); return IOStatus::OK();
} }
Status WinMmapFile::Allocate(uint64_t offset, uint64_t len) { IOStatus WinMmapFile::Allocate(uint64_t offset, uint64_t len,
Status status; const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
IOStatus status;
TEST_KILL_RANDOM("WinMmapFile::Allocate", rocksdb_kill_odds); TEST_KILL_RANDOM("WinMmapFile::Allocate", rocksdb_kill_odds);
// Make sure that we reserve an aligned amount of space // Make sure that we reserve an aligned amount of space
// since the reservation block size is driven outside so we want // since the reservation block size is driven outside so we want
// to check if we are ok with reservation here // to check if we are ok with reservation here
size_t spaceToReserve = Roundup(static_cast<size_t>(offset + len), view_size_); size_t spaceToReserve =
Roundup(static_cast<size_t>(offset + len), view_size_);
// Nothing to do // Nothing to do
if (spaceToReserve <= reserved_size_) { if (spaceToReserve <= reserved_size_) {
return status; return status;
@ -563,31 +567,34 @@ size_t WinMmapFile::GetUniqueId(char* id, size_t max_size) const {
// WinSequentialFile // WinSequentialFile
WinSequentialFile::WinSequentialFile(const std::string& fname, HANDLE f, WinSequentialFile::WinSequentialFile(const std::string& fname, HANDLE f,
const EnvOptions& options) const FileOptions& options)
: WinFileData(fname, f, options.use_direct_reads) {} : WinFileData(fname, f, options.use_direct_reads) {}
WinSequentialFile::~WinSequentialFile() { WinSequentialFile::~WinSequentialFile() {
assert(hFile_ != INVALID_HANDLE_VALUE); assert(hFile_ != INVALID_HANDLE_VALUE);
} }
Status WinSequentialFile::Read(size_t n, Slice* result, char* scratch) { IOStatus WinSequentialFile::Read(size_t n, const IOOptions& /*opts*/,
Status s; Slice* result, char* scratch,
IODebugContext* /*dbg*/) {
IOStatus s;
size_t r = 0; size_t r = 0;
assert(result != nullptr); assert(result != nullptr);
if (WinFileData::use_direct_io()) { if (WinFileData::use_direct_io()) {
return Status::NotSupported("Read() does not support direct_io"); return IOStatus::NotSupported("Read() does not support direct_io");
} }
// Windows ReadFile API accepts a DWORD. // Windows ReadFile API accepts a DWORD.
// While it is possible to read in a loop if n is too big // While it is possible to read in a loop if n is too big
// it is an unlikely case. // it is an unlikely case.
if (n > std::numeric_limits<DWORD>::max()) { if (n > std::numeric_limits<DWORD>::max()) {
return Status::InvalidArgument("n is too big for a single ReadFile: " return IOStatus::InvalidArgument("n is too big for a single ReadFile: " +
+ filename_); filename_);
} }
DWORD bytesToRead = static_cast<DWORD>(n); //cast is safe due to the check above DWORD bytesToRead =
static_cast<DWORD>(n); // cast is safe due to the check above
DWORD bytesRead = 0; DWORD bytesRead = 0;
BOOL ret = ReadFile(hFile_, scratch, bytesToRead, &bytesRead, NULL); BOOL ret = ReadFile(hFile_, scratch, bytesToRead, &bytesRead, NULL);
if (ret != FALSE) { if (ret != FALSE) {
@ -595,8 +602,7 @@ Status WinSequentialFile::Read(size_t n, Slice* result, char* scratch) {
} else { } else {
auto lastError = GetLastError(); auto lastError = GetLastError();
if (lastError != ERROR_HANDLE_EOF) { if (lastError != ERROR_HANDLE_EOF) {
s = IOErrorFromWindowsError("ReadFile failed: " + filename_, s = IOErrorFromWindowsError("ReadFile failed: " + filename_, lastError);
lastError);
} }
} }
@ -604,99 +610,91 @@ Status WinSequentialFile::Read(size_t n, Slice* result, char* scratch) {
return s; return s;
} }
Status WinSequentialFile::PositionedReadInternal(char* src, size_t numBytes, IOStatus WinSequentialFile::PositionedReadInternal(char* src, size_t numBytes,
uint64_t offset, size_t& bytes_read) const { uint64_t offset,
size_t& bytes_read) const {
return pread(this, src, numBytes, offset, bytes_read); return pread(this, src, numBytes, offset, bytes_read);
} }
Status WinSequentialFile::PositionedRead(uint64_t offset, size_t n, Slice* result, IOStatus WinSequentialFile::PositionedRead(uint64_t offset, size_t n,
char* scratch) { const IOOptions& /*opts*/,
Slice* result, char* scratch,
Status s; IODebugContext* /*dbg*/) {
if (!WinFileData::use_direct_io()) { if (!WinFileData::use_direct_io()) {
return Status::NotSupported("This function is only used for direct_io"); return IOStatus::NotSupported("This function is only used for direct_io");
} }
if (!IsSectorAligned(static_cast<size_t>(offset)) || if (!IsSectorAligned(static_cast<size_t>(offset)) || !IsSectorAligned(n)) {
!IsSectorAligned(n)) { return IOStatus::InvalidArgument(
return Status::InvalidArgument(
"WinSequentialFile::PositionedRead: offset is not properly aligned"); "WinSequentialFile::PositionedRead: offset is not properly aligned");
} }
size_t bytes_read = 0; // out param size_t bytes_read = 0; // out param
s = PositionedReadInternal(scratch, static_cast<size_t>(n), offset, bytes_read); IOStatus s = PositionedReadInternal(scratch, static_cast<size_t>(n), offset,
bytes_read);
*result = Slice(scratch, bytes_read); *result = Slice(scratch, bytes_read);
return s; return s;
} }
IOStatus WinSequentialFile::Skip(uint64_t n) {
Status WinSequentialFile::Skip(uint64_t n) { // Can't handle more than signed max as SetFilePointerEx accepts a signed
// Can't handle more than signed max as SetFilePointerEx accepts a signed 64-bit // 64-bit integer. As such it is a highly unlikley case to have n so large.
// integer. As such it is a highly unlikley case to have n so large.
if (n > static_cast<uint64_t>(std::numeric_limits<LONGLONG>::max())) { if (n > static_cast<uint64_t>(std::numeric_limits<LONGLONG>::max())) {
return Status::InvalidArgument("n is too large for a single SetFilePointerEx() call" + return IOStatus::InvalidArgument(
filename_); "n is too large for a single SetFilePointerEx() call" + filename_);
} }
LARGE_INTEGER li; LARGE_INTEGER li;
li.QuadPart = static_cast<LONGLONG>(n); //cast is safe due to the check above li.QuadPart = static_cast<LONGLONG>(n); // cast is safe due to the check
// above
BOOL ret = SetFilePointerEx(hFile_, li, NULL, FILE_CURRENT); BOOL ret = SetFilePointerEx(hFile_, li, NULL, FILE_CURRENT);
if (ret == FALSE) { if (ret == FALSE) {
auto lastError = GetLastError(); auto lastError = GetLastError();
return IOErrorFromWindowsError("Skip SetFilePointerEx():" + filename_, return IOErrorFromWindowsError("Skip SetFilePointerEx():" + filename_,
lastError); lastError);
} }
return Status::OK(); return IOStatus::OK();
} }
Status WinSequentialFile::InvalidateCache(size_t offset, size_t length) { IOStatus WinSequentialFile::InvalidateCache(size_t offset, size_t length) {
return Status::OK(); return IOStatus::OK();
} }
////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////
/// WinRandomAccessBase /// WinRandomAccessBase
inline inline IOStatus WinRandomAccessImpl::PositionedReadInternal(
Status WinRandomAccessImpl::PositionedReadInternal(char* src, char* src, size_t numBytes, uint64_t offset, size_t& bytes_read) const {
size_t numBytes,
uint64_t offset,
size_t& bytes_read) const {
return pread(file_base_, src, numBytes, offset, bytes_read); return pread(file_base_, src, numBytes, offset, bytes_read);
} }
inline inline WinRandomAccessImpl::WinRandomAccessImpl(WinFileData* file_base,
WinRandomAccessImpl::WinRandomAccessImpl(WinFileData* file_base,
size_t alignment, size_t alignment,
const EnvOptions& options) : const FileOptions& options)
file_base_(file_base), : file_base_(file_base), alignment_(alignment) {
alignment_(alignment) {
assert(!options.use_mmap_reads); assert(!options.use_mmap_reads);
} }
inline inline IOStatus WinRandomAccessImpl::ReadImpl(uint64_t offset, size_t n,
Status WinRandomAccessImpl::ReadImpl(uint64_t offset, size_t n, Slice* result, Slice* result,
char* scratch) const { char* scratch) const {
Status s;
// Check buffer alignment // Check buffer alignment
if (file_base_->use_direct_io()) { if (file_base_->use_direct_io()) {
if (!IsSectorAligned(static_cast<size_t>(offset)) || if (!IsSectorAligned(static_cast<size_t>(offset)) ||
!IsAligned(alignment_, scratch)) { !IsAligned(alignment_, scratch)) {
return Status::InvalidArgument( return IOStatus::InvalidArgument(
"WinRandomAccessImpl::ReadImpl: offset or scratch is not properly aligned"); "WinRandomAccessImpl::ReadImpl: offset or scratch is not properly "
"aligned");
} }
} }
if (n == 0) { if (n == 0) {
*result = Slice(scratch, 0); *result = Slice(scratch, 0);
return s; return IOStatus::OK();
} }
size_t bytes_read = 0; size_t bytes_read = 0;
s = PositionedReadInternal(scratch, n, offset, bytes_read); IOStatus s = PositionedReadInternal(scratch, n, offset, bytes_read);
*result = Slice(scratch, bytes_read); *result = Slice(scratch, bytes_read);
return s; return s;
} }
@ -706,20 +704,21 @@ Status WinRandomAccessImpl::ReadImpl(uint64_t offset, size_t n, Slice* result,
WinRandomAccessFile::WinRandomAccessFile(const std::string& fname, HANDLE hFile, WinRandomAccessFile::WinRandomAccessFile(const std::string& fname, HANDLE hFile,
size_t alignment, size_t alignment,
const EnvOptions& options) const FileOptions& options)
: WinFileData(fname, hFile, options.use_direct_reads), : WinFileData(fname, hFile, options.use_direct_reads),
WinRandomAccessImpl(this, alignment, options) {} WinRandomAccessImpl(this, alignment, options) {}
WinRandomAccessFile::~WinRandomAccessFile() { WinRandomAccessFile::~WinRandomAccessFile() {}
}
Status WinRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result, IOStatus WinRandomAccessFile::Read(uint64_t offset, size_t n,
char* scratch) const { const IOOptions& /*options*/, Slice* result,
char* scratch,
IODebugContext* /*dbg*/) const {
return ReadImpl(offset, n, result, scratch); return ReadImpl(offset, n, result, scratch);
} }
Status WinRandomAccessFile::InvalidateCache(size_t offset, size_t length) { IOStatus WinRandomAccessFile::InvalidateCache(size_t offset, size_t length) {
return Status::OK(); return IOStatus::OK();
} }
size_t WinRandomAccessFile::GetUniqueId(char* id, size_t max_size) const { size_t WinRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
@ -734,18 +733,17 @@ size_t WinRandomAccessFile::GetRequiredBufferAlignment() const {
// WinWritableImpl // WinWritableImpl
// //
inline inline IOStatus WinWritableImpl::PreallocateInternal(uint64_t spaceToReserve) {
Status WinWritableImpl::PreallocateInternal(uint64_t spaceToReserve) { return fallocate(file_data_->GetName(), file_data_->GetFileHandle(),
return fallocate(file_data_->GetName(), file_data_->GetFileHandle(), spaceToReserve); spaceToReserve);
} }
inline inline WinWritableImpl::WinWritableImpl(WinFileData* file_data,
WinWritableImpl::WinWritableImpl(WinFileData* file_data, size_t alignment) size_t alignment)
: file_data_(file_data), : file_data_(file_data),
alignment_(alignment), alignment_(alignment),
next_write_offset_(0), next_write_offset_(0),
reservedsize_(0) { reservedsize_(0) {
// Query current position in case ReopenWritableFile is called // Query current position in case ReopenWritableFile is called
// This position is only important for buffered writes // This position is only important for buffered writes
// for unbuffered writes we explicitely specify the position. // for unbuffered writes we explicitely specify the position.
@ -763,13 +761,11 @@ WinWritableImpl::WinWritableImpl(WinFileData* file_data, size_t alignment)
} }
} }
inline inline IOStatus WinWritableImpl::AppendImpl(const Slice& data) {
Status WinWritableImpl::AppendImpl(const Slice& data) { IOStatus s;
Status s;
if (data.size() > std::numeric_limits<DWORD>::max()) { if (data.size() > std::numeric_limits<DWORD>::max()) {
return Status::InvalidArgument("data is too long for a single write" + return IOStatus::InvalidArgument("data is too long for a single write" +
file_data_->GetName()); file_data_->GetName());
} }
@ -781,33 +777,31 @@ Status WinWritableImpl::AppendImpl(const Slice& data) {
assert(IsSectorAligned(next_write_offset_)); assert(IsSectorAligned(next_write_offset_));
if (!IsSectorAligned(data.size()) || if (!IsSectorAligned(data.size()) ||
!IsAligned(static_cast<size_t>(GetAlignement()), data.data())) { !IsAligned(static_cast<size_t>(GetAlignement()), data.data())) {
s = Status::InvalidArgument( s = IOStatus::InvalidArgument(
"WriteData must be page aligned, size must be sector aligned"); "WriteData must be page aligned, size must be sector aligned");
} else { } else {
s = pwrite(file_data_, data, next_write_offset_, bytes_written); s = pwrite(file_data_, data, next_write_offset_, bytes_written);
} }
} else { } else {
DWORD bytesWritten = 0; DWORD bytesWritten = 0;
if (!WriteFile(file_data_->GetFileHandle(), data.data(), if (!WriteFile(file_data_->GetFileHandle(), data.data(),
static_cast<DWORD>(data.size()), &bytesWritten, NULL)) { static_cast<DWORD>(data.size()), &bytesWritten, NULL)) {
auto lastError = GetLastError(); auto lastError = GetLastError();
s = IOErrorFromWindowsError( s = IOErrorFromWindowsError(
"Failed to WriteFile: " + file_data_->GetName(), "Failed to WriteFile: " + file_data_->GetName(), lastError);
lastError);
} else { } else {
bytes_written = bytesWritten; bytes_written = bytesWritten;
} }
} }
if(s.ok()) { if (s.ok()) {
if (bytes_written == data.size()) { if (bytes_written == data.size()) {
// This matters for direct_io cases where // This matters for direct_io cases where
// we rely on the fact that next_write_offset_ // we rely on the fact that next_write_offset_
// is sector aligned // is sector aligned
next_write_offset_ += bytes_written; next_write_offset_ += bytes_written;
} else { } else {
s = Status::IOError("Failed to write all bytes: " + s = IOStatus::IOError("Failed to write all bytes: " +
file_data_->GetName()); file_data_->GetName());
} }
} }
@ -815,22 +809,21 @@ Status WinWritableImpl::AppendImpl(const Slice& data) {
return s; return s;
} }
inline inline IOStatus WinWritableImpl::PositionedAppendImpl(const Slice& data,
Status WinWritableImpl::PositionedAppendImpl(const Slice& data, uint64_t offset) { uint64_t offset) {
if (file_data_->use_direct_io()) {
if(file_data_->use_direct_io()) {
if (!IsSectorAligned(static_cast<size_t>(offset)) || if (!IsSectorAligned(static_cast<size_t>(offset)) ||
!IsSectorAligned(data.size()) || !IsSectorAligned(data.size()) ||
!IsAligned(static_cast<size_t>(GetAlignement()), data.data())) { !IsAligned(static_cast<size_t>(GetAlignement()), data.data())) {
return Status::InvalidArgument( return IOStatus::InvalidArgument(
"Data and offset must be page aligned, size must be sector aligned"); "Data and offset must be page aligned, size must be sector aligned");
} }
} }
size_t bytes_written = 0; size_t bytes_written = 0;
Status s = pwrite(file_data_, data, offset, bytes_written); IOStatus s = pwrite(file_data_, data, offset, bytes_written);
if(s.ok()) { if (s.ok()) {
if (bytes_written == data.size()) { if (bytes_written == data.size()) {
// For sequential write this would be simple // For sequential write this would be simple
// size extension by data.size() // size extension by data.size()
@ -839,23 +832,21 @@ Status WinWritableImpl::PositionedAppendImpl(const Slice& data, uint64_t offset)
next_write_offset_ = write_end; next_write_offset_ = write_end;
} }
} else { } else {
s = Status::IOError("Failed to write all of the requested data: " + s = IOStatus::IOError("Failed to write all of the requested data: " +
file_data_->GetName()); file_data_->GetName());
} }
} }
return s; return s;
} }
inline inline IOStatus WinWritableImpl::TruncateImpl(uint64_t size) {
Status WinWritableImpl::TruncateImpl(uint64_t size) {
// It is tempting to check for the size for sector alignment // It is tempting to check for the size for sector alignment
// but truncation may come at the end and there is not a requirement // but truncation may come at the end and there is not a requirement
// for this to be sector aligned so long as we do not attempt to write // for this to be sector aligned so long as we do not attempt to write
// after that. The interface docs state that the behavior is undefined // after that. The interface docs state that the behavior is undefined
// in that case. // in that case.
Status s = ftruncate(file_data_->GetName(), file_data_->GetFileHandle(), IOStatus s =
size); ftruncate(file_data_->GetName(), file_data_->GetFileHandle(), size);
if (s.ok()) { if (s.ok()) {
next_write_offset_ = size; next_write_offset_ = size;
@ -863,50 +854,48 @@ Status WinWritableImpl::TruncateImpl(uint64_t size) {
return s; return s;
} }
inline inline IOStatus WinWritableImpl::CloseImpl() {
Status WinWritableImpl::CloseImpl() { IOStatus s;
Status s;
auto hFile = file_data_->GetFileHandle(); auto hFile = file_data_->GetFileHandle();
assert(INVALID_HANDLE_VALUE != hFile); assert(INVALID_HANDLE_VALUE != hFile);
if (!::FlushFileBuffers(hFile)) { if (!::FlushFileBuffers(hFile)) {
auto lastError = GetLastError(); auto lastError = GetLastError();
s = IOErrorFromWindowsError("FlushFileBuffers failed at Close() for: " + s = IOErrorFromWindowsError(
file_data_->GetName(), "FlushFileBuffers failed at Close() for: " + file_data_->GetName(),
lastError); lastError);
} }
if(!file_data_->CloseFile() && s.ok()) { if (!file_data_->CloseFile() && s.ok()) {
auto lastError = GetLastError();
s = IOErrorFromWindowsError("CloseHandle failed for: " + file_data_->GetName(),
lastError);
}
return s;
}
inline
Status WinWritableImpl::SyncImpl() {
Status s;
if (!::FlushFileBuffers (file_data_->GetFileHandle())) {
auto lastError = GetLastError(); auto lastError = GetLastError();
s = IOErrorFromWindowsError( s = IOErrorFromWindowsError(
"FlushFileBuffers failed at Sync() for: " + file_data_->GetName(), lastError); "CloseHandle failed for: " + file_data_->GetName(), lastError);
} }
return s; return s;
} }
inline IOStatus WinWritableImpl::SyncImpl(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
IOStatus s;
if (!::FlushFileBuffers(file_data_->GetFileHandle())) {
auto lastError = GetLastError();
s = IOErrorFromWindowsError(
"FlushFileBuffers failed at Sync() for: " + file_data_->GetName(),
lastError);
}
return s;
}
inline inline IOStatus WinWritableImpl::AllocateImpl(uint64_t offset, uint64_t len) {
Status WinWritableImpl::AllocateImpl(uint64_t offset, uint64_t len) { IOStatus status;
Status status;
TEST_KILL_RANDOM("WinWritableFile::Allocate", rocksdb_kill_odds); TEST_KILL_RANDOM("WinWritableFile::Allocate", rocksdb_kill_odds);
// Make sure that we reserve an aligned amount of space // Make sure that we reserve an aligned amount of space
// since the reservation block size is driven outside so we want // since the reservation block size is driven outside so we want
// to check if we are ok with reservation here // to check if we are ok with reservation here
size_t spaceToReserve = Roundup(static_cast<size_t>(offset + len), static_cast<size_t>(alignment_)); size_t spaceToReserve = Roundup(static_cast<size_t>(offset + len),
static_cast<size_t>(alignment_));
// Nothing to do // Nothing to do
if (spaceToReserve <= reservedsize_) { if (spaceToReserve <= reservedsize_) {
return status; return status;
@ -920,66 +909,78 @@ Status WinWritableImpl::AllocateImpl(uint64_t offset, uint64_t len) {
return status; return status;
} }
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/// WinWritableFile /// WinWritableFile
WinWritableFile::WinWritableFile(const std::string& fname, HANDLE hFile, WinWritableFile::WinWritableFile(const std::string& fname, HANDLE hFile,
size_t alignment, size_t /* capacity */, size_t alignment, size_t /* capacity */,
const EnvOptions& options) const FileOptions& options)
: WinFileData(fname, hFile, options.use_direct_writes), : WinFileData(fname, hFile, options.use_direct_writes),
WinWritableImpl(this, alignment), WinWritableImpl(this, alignment),
WritableFile(options) { FSWritableFile(options) {
assert(!options.use_mmap_writes); assert(!options.use_mmap_writes);
} }
WinWritableFile::~WinWritableFile() { WinWritableFile::~WinWritableFile() {}
}
// Indicates if the class makes use of direct I/O // Indicates if the class makes use of direct I/O
bool WinWritableFile::use_direct_io() const { return WinFileData::use_direct_io(); } bool WinWritableFile::use_direct_io() const {
return WinFileData::use_direct_io();
}
size_t WinWritableFile::GetRequiredBufferAlignment() const { size_t WinWritableFile::GetRequiredBufferAlignment() const {
return static_cast<size_t>(GetAlignement()); return static_cast<size_t>(GetAlignement());
} }
Status WinWritableFile::Append(const Slice& data) { IOStatus WinWritableFile::Append(const Slice& data,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return AppendImpl(data); return AppendImpl(data);
} }
Status WinWritableFile::PositionedAppend(const Slice& data, uint64_t offset) { IOStatus WinWritableFile::PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return PositionedAppendImpl(data, offset); return PositionedAppendImpl(data, offset);
} }
// Need to implement this so the file is truncated correctly // Need to implement this so the file is truncated correctly
// when buffered and unbuffered mode // when buffered and unbuffered mode
Status WinWritableFile::Truncate(uint64_t size) { IOStatus WinWritableFile::Truncate(uint64_t size, const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return TruncateImpl(size); return TruncateImpl(size);
} }
Status WinWritableFile::Close() { IOStatus WinWritableFile::Close(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return CloseImpl(); return CloseImpl();
} }
// write out the cached data to the OS cache // write out the cached data to the OS cache
// This is now taken care of the WritableFileWriter // This is now taken care of the WritableFileWriter
Status WinWritableFile::Flush() { IOStatus WinWritableFile::Flush(const IOOptions& /*options*/,
return Status::OK(); IODebugContext* /*dbg*/) {
return IOStatus::OK();
} }
Status WinWritableFile::Sync() { IOStatus WinWritableFile::Sync(const IOOptions& options, IODebugContext* dbg) {
return SyncImpl(); return SyncImpl(options, dbg);
} }
Status WinWritableFile::Fsync() { return SyncImpl(); } IOStatus WinWritableFile::Fsync(const IOOptions& options, IODebugContext* dbg) {
return SyncImpl(options, dbg);
}
bool WinWritableFile::IsSyncThreadSafe() const { return true; } bool WinWritableFile::IsSyncThreadSafe() const { return true; }
uint64_t WinWritableFile::GetFileSize() { uint64_t WinWritableFile::GetFileSize(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return GetFileNextWriteOffset(); return GetFileNextWriteOffset();
} }
Status WinWritableFile::Allocate(uint64_t offset, uint64_t len) { IOStatus WinWritableFile::Allocate(uint64_t offset, uint64_t len,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return AllocateImpl(offset, len); return AllocateImpl(offset, len);
} }
@ -991,36 +992,43 @@ size_t WinWritableFile::GetUniqueId(char* id, size_t max_size) const {
/// WinRandomRWFile /// WinRandomRWFile
WinRandomRWFile::WinRandomRWFile(const std::string& fname, HANDLE hFile, WinRandomRWFile::WinRandomRWFile(const std::string& fname, HANDLE hFile,
size_t alignment, const EnvOptions& options) size_t alignment, const FileOptions& options)
: WinFileData(fname, hFile, : WinFileData(fname, hFile,
options.use_direct_reads && options.use_direct_writes), options.use_direct_reads && options.use_direct_writes),
WinRandomAccessImpl(this, alignment, options), WinRandomAccessImpl(this, alignment, options),
WinWritableImpl(this, alignment) {} WinWritableImpl(this, alignment) {}
bool WinRandomRWFile::use_direct_io() const { return WinFileData::use_direct_io(); } bool WinRandomRWFile::use_direct_io() const {
return WinFileData::use_direct_io();
}
size_t WinRandomRWFile::GetRequiredBufferAlignment() const { size_t WinRandomRWFile::GetRequiredBufferAlignment() const {
return static_cast<size_t>(GetAlignement()); return static_cast<size_t>(GetAlignement());
} }
Status WinRandomRWFile::Write(uint64_t offset, const Slice & data) { IOStatus WinRandomRWFile::Write(uint64_t offset, const Slice& data,
const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return PositionedAppendImpl(data, offset); return PositionedAppendImpl(data, offset);
} }
Status WinRandomRWFile::Read(uint64_t offset, size_t n, Slice* result, IOStatus WinRandomRWFile::Read(uint64_t offset, size_t n,
char* scratch) const { const IOOptions& /*options*/, Slice* result,
char* scratch, IODebugContext* /*dbg*/) const {
return ReadImpl(offset, n, result, scratch); return ReadImpl(offset, n, result, scratch);
} }
Status WinRandomRWFile::Flush() { IOStatus WinRandomRWFile::Flush(const IOOptions& /*options*/,
return Status::OK(); IODebugContext* /*dbg*/) {
return IOStatus::OK();
} }
Status WinRandomRWFile::Sync() { IOStatus WinRandomRWFile::Sync(const IOOptions& options, IODebugContext* dbg) {
return SyncImpl(); return SyncImpl(options, dbg);
} }
Status WinRandomRWFile::Close() { IOStatus WinRandomRWFile::Close(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return CloseImpl(); return CloseImpl();
} }
@ -1053,7 +1061,10 @@ WinMemoryMappedBuffer::~WinMemoryMappedBuffer() {
////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////
/// WinDirectory /// WinDirectory
Status WinDirectory::Fsync() { return Status::OK(); } IOStatus WinDirectory::Fsync(const IOOptions& /*options*/,
IODebugContext* /*dbg*/) {
return IOStatus::OK();
}
size_t WinDirectory::GetUniqueId(char* id, size_t max_size) const { size_t WinDirectory::GetUniqueId(char* id, size_t max_size) const {
return GetUniqueIdFromFile(handle_, id, max_size); return GetUniqueIdFromFile(handle_, id, max_size);
@ -1067,7 +1078,7 @@ WinFileLock::~WinFileLock() {
assert(ret); assert(ret);
} }
} } // namespace port
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE
#endif #endif

View File

@ -9,51 +9,51 @@
#pragma once #pragma once
#include <stdint.h> #include <stdint.h>
#include <windows.h>
#include <mutex> #include <mutex>
#include <string> #include <string>
#include "rocksdb/file_system.h"
#include "rocksdb/status.h" #include "rocksdb/status.h"
#include "rocksdb/env.h"
#include "util/aligned_buffer.h" #include "util/aligned_buffer.h"
#include <windows.h>
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace port { namespace port {
std::string GetWindowsErrSz(DWORD err); std::string GetWindowsErrSz(DWORD err);
inline Status IOErrorFromWindowsError(const std::string& context, DWORD err) { inline IOStatus IOErrorFromWindowsError(const std::string& context, DWORD err) {
return ((err == ERROR_HANDLE_DISK_FULL) || (err == ERROR_DISK_FULL)) return ((err == ERROR_HANDLE_DISK_FULL) || (err == ERROR_DISK_FULL))
? Status::NoSpace(context, GetWindowsErrSz(err)) ? IOStatus::NoSpace(context, GetWindowsErrSz(err))
: ((err == ERROR_FILE_NOT_FOUND) || (err == ERROR_PATH_NOT_FOUND)) : ((err == ERROR_FILE_NOT_FOUND) || (err == ERROR_PATH_NOT_FOUND))
? Status::PathNotFound(context, GetWindowsErrSz(err)) ? IOStatus::PathNotFound(context, GetWindowsErrSz(err))
: Status::IOError(context, GetWindowsErrSz(err)); : IOStatus::IOError(context, GetWindowsErrSz(err));
} }
inline Status IOErrorFromLastWindowsError(const std::string& context) { inline IOStatus IOErrorFromLastWindowsError(const std::string& context) {
return IOErrorFromWindowsError(context, GetLastError()); return IOErrorFromWindowsError(context, GetLastError());
} }
inline Status IOError(const std::string& context, int err_number) { inline IOStatus IOError(const std::string& context, int err_number) {
return (err_number == ENOSPC) return (err_number == ENOSPC)
? Status::NoSpace(context, strerror(err_number)) ? IOStatus::NoSpace(context, strerror(err_number))
: (err_number == ENOENT) : (err_number == ENOENT)
? Status::PathNotFound(context, strerror(err_number)) ? IOStatus::PathNotFound(context, strerror(err_number))
: Status::IOError(context, strerror(err_number)); : IOStatus::IOError(context, strerror(err_number));
} }
class WinFileData; class WinFileData;
Status pwrite(const WinFileData* file_data, const Slice& data, IOStatus pwrite(const WinFileData* file_data, const Slice& data,
uint64_t offset, size_t& bytes_written); uint64_t offset, size_t& bytes_written);
Status pread(const WinFileData* file_data, char* src, size_t num_bytes, IOStatus pread(const WinFileData* file_data, char* src, size_t num_bytes,
uint64_t offset, size_t& bytes_read); uint64_t offset, size_t& bytes_read);
Status fallocate(const std::string& filename, HANDLE hFile, uint64_t to_size); IOStatus fallocate(const std::string& filename, HANDLE hFile, uint64_t to_size);
Status ftruncate(const std::string& filename, HANDLE hFile, uint64_t toSize); IOStatus ftruncate(const std::string& filename, HANDLE hFile, uint64_t toSize);
size_t GetUniqueIdFromFile(HANDLE hFile, char* id, size_t max_size); size_t GetUniqueIdFromFile(HANDLE hFile, char* id, size_t max_size);
@ -95,34 +95,38 @@ class WinFileData {
WinFileData& operator=(const WinFileData&) = delete; WinFileData& operator=(const WinFileData&) = delete;
}; };
class WinSequentialFile : protected WinFileData, public SequentialFile { class WinSequentialFile : protected WinFileData, public FSSequentialFile {
// Override for behavior change when creating a custom env // Override for behavior change when creating a custom env
virtual Status PositionedReadInternal(char* src, size_t numBytes, virtual IOStatus PositionedReadInternal(char* src, size_t numBytes,
uint64_t offset, size_t& bytes_read) const; uint64_t offset,
size_t& bytes_read) const;
public: public:
WinSequentialFile(const std::string& fname, HANDLE f, WinSequentialFile(const std::string& fname, HANDLE f,
const EnvOptions& options); const FileOptions& options);
~WinSequentialFile(); ~WinSequentialFile();
WinSequentialFile(const WinSequentialFile&) = delete; WinSequentialFile(const WinSequentialFile&) = delete;
WinSequentialFile& operator=(const WinSequentialFile&) = delete; WinSequentialFile& operator=(const WinSequentialFile&) = delete;
virtual Status Read(size_t n, Slice* result, char* scratch) override; IOStatus Read(size_t n, const IOOptions& options, Slice* result,
virtual Status PositionedRead(uint64_t offset, size_t n, Slice* result, char* scratch, IODebugContext* dbg) override;
char* scratch) override; IOStatus PositionedRead(uint64_t offset, size_t n, const IOOptions& options,
Slice* result, char* scratch,
IODebugContext* dbg) override;
virtual Status Skip(uint64_t n) override; IOStatus Skip(uint64_t n) override;
virtual Status InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
virtual bool use_direct_io() const override { return WinFileData::use_direct_io(); } virtual bool use_direct_io() const override {
return WinFileData::use_direct_io();
}
}; };
// mmap() based random-access // mmap() based random-access
class WinMmapReadableFile : private WinFileData, public RandomAccessFile { class WinMmapReadableFile : private WinFileData, public FSRandomAccessFile {
HANDLE hMap_; HANDLE hMap_;
const void* mapped_region_; const void* mapped_region_;
@ -138,10 +142,11 @@ class WinMmapReadableFile : private WinFileData, public RandomAccessFile {
WinMmapReadableFile(const WinMmapReadableFile&) = delete; WinMmapReadableFile(const WinMmapReadableFile&) = delete;
WinMmapReadableFile& operator=(const WinMmapReadableFile&) = delete; WinMmapReadableFile& operator=(const WinMmapReadableFile&) = delete;
virtual Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) const override; Slice* result, char* scratch,
IODebugContext* dbg) const override;
virtual Status InvalidateCache(size_t offset, size_t length) override; virtual IOStatus InvalidateCache(size_t offset, size_t length) override;
virtual size_t GetUniqueId(char* id, size_t max_size) const override; virtual size_t GetUniqueId(char* id, size_t max_size) const override;
}; };
@ -150,7 +155,7 @@ class WinMmapReadableFile : private WinFileData, public RandomAccessFile {
// data to the file. This is safe since we either properly close the // data to the file. This is safe since we either properly close the
// file before reading from it, or for log files, the reading code // file before reading from it, or for log files, the reading code
// knows enough to skip zero suffixes. // knows enough to skip zero suffixes.
class WinMmapFile : private WinFileData, public WritableFile { class WinMmapFile : private WinFileData, public FSWritableFile {
private: private:
HANDLE hMap_; HANDLE hMap_;
@ -179,51 +184,59 @@ class WinMmapFile : private WinFileData, public WritableFile {
// Can only truncate or reserve to a sector size aligned if // Can only truncate or reserve to a sector size aligned if
// used on files that are opened with Unbuffered I/O // used on files that are opened with Unbuffered I/O
Status TruncateFile(uint64_t toSize); IOStatus TruncateFile(uint64_t toSize);
Status UnmapCurrentRegion(); IOStatus UnmapCurrentRegion();
Status MapNewRegion(); IOStatus MapNewRegion(const IOOptions& options, IODebugContext* dbg);
virtual Status PreallocateInternal(uint64_t spaceToReserve); virtual IOStatus PreallocateInternal(uint64_t spaceToReserve);
public: public:
WinMmapFile(const std::string& fname, HANDLE hFile, size_t page_size, WinMmapFile(const std::string& fname, HANDLE hFile, size_t page_size,
size_t allocation_granularity, const EnvOptions& options); size_t allocation_granularity, const FileOptions& options);
~WinMmapFile(); ~WinMmapFile();
WinMmapFile(const WinMmapFile&) = delete; WinMmapFile(const WinMmapFile&) = delete;
WinMmapFile& operator=(const WinMmapFile&) = delete; WinMmapFile& operator=(const WinMmapFile&) = delete;
virtual Status Append(const Slice& data) override; IOStatus Append(const Slice& data, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus Append(const Slice& data, const IOOptions& opts,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) override {
return Append(data, opts, dbg);
}
// Means Close() will properly take care of truncate // Means Close() will properly take care of truncate
// and it does not need any additional information // and it does not need any additional information
virtual Status Truncate(uint64_t size) override; IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override;
virtual Status Close() override; IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Flush() override; IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override;
// Flush only data // Flush only data
virtual Status Sync() override; IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
/** /**
* Flush data as well as metadata to stable storage. * Flush data as well as metadata to stable storage.
*/ */
virtual Status Fsync() override; IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override;
/** /**
* Get the size of valid data in the file. This will not match the * Get the size of valid data in the file. This will not match the
* size that is returned from the filesystem because we use mmap * size that is returned from the filesystem because we use mmap
* to extend file by map_size every time. * to extend file by map_size every time.
*/ */
virtual uint64_t GetFileSize() override; uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override;
virtual Status InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
virtual Status Allocate(uint64_t offset, uint64_t len) override; IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options,
IODebugContext* dbg) override;
virtual size_t GetUniqueId(char* id, size_t max_size) const override; virtual size_t GetUniqueId(char* id, size_t max_size) const override;
}; };
@ -234,21 +247,21 @@ class WinRandomAccessImpl {
size_t alignment_; size_t alignment_;
// Override for behavior change when creating a custom env // Override for behavior change when creating a custom env
virtual Status PositionedReadInternal(char* src, size_t numBytes, virtual IOStatus PositionedReadInternal(char* src, size_t numBytes,
uint64_t offset, size_t& bytes_read) const; uint64_t offset,
size_t& bytes_read) const;
WinRandomAccessImpl(WinFileData* file_base, size_t alignment, WinRandomAccessImpl(WinFileData* file_base, size_t alignment,
const EnvOptions& options); const FileOptions& options);
virtual ~WinRandomAccessImpl() {} virtual ~WinRandomAccessImpl() {}
Status ReadImpl(uint64_t offset, size_t n, Slice* result, IOStatus ReadImpl(uint64_t offset, size_t n, Slice* result,
char* scratch) const; char* scratch) const;
size_t GetAlignment() const { return alignment_; } size_t GetAlignment() const { return alignment_; }
public: public:
WinRandomAccessImpl(const WinRandomAccessImpl&) = delete; WinRandomAccessImpl(const WinRandomAccessImpl&) = delete;
WinRandomAccessImpl& operator=(const WinRandomAccessImpl&) = delete; WinRandomAccessImpl& operator=(const WinRandomAccessImpl&) = delete;
}; };
@ -258,21 +271,24 @@ class WinRandomAccessFile
: private WinFileData, : private WinFileData,
protected WinRandomAccessImpl, // Want to be able to override protected WinRandomAccessImpl, // Want to be able to override
// PositionedReadInternal // PositionedReadInternal
public RandomAccessFile { public FSRandomAccessFile {
public: public:
WinRandomAccessFile(const std::string& fname, HANDLE hFile, size_t alignment, WinRandomAccessFile(const std::string& fname, HANDLE hFile, size_t alignment,
const EnvOptions& options); const FileOptions& options);
~WinRandomAccessFile(); ~WinRandomAccessFile();
virtual Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) const override; Slice* result, char* scratch,
IODebugContext* dbg) const override;
virtual size_t GetUniqueId(char* id, size_t max_size) const override; virtual size_t GetUniqueId(char* id, size_t max_size) const override;
virtual bool use_direct_io() const override { return WinFileData::use_direct_io(); } virtual bool use_direct_io() const override {
return WinFileData::use_direct_io();
}
virtual Status InvalidateCache(size_t offset, size_t length) override; IOStatus InvalidateCache(size_t offset, size_t length) override;
virtual size_t GetRequiredBufferAlignment() const override; virtual size_t GetRequiredBufferAlignment() const override;
}; };
@ -293,10 +309,11 @@ class WinWritableImpl {
protected: protected:
WinFileData* file_data_; WinFileData* file_data_;
const uint64_t alignment_; const uint64_t alignment_;
uint64_t next_write_offset_; // Needed because Windows does not support O_APPEND uint64_t
next_write_offset_; // Needed because Windows does not support O_APPEND
uint64_t reservedsize_; // how far we have reserved space uint64_t reservedsize_; // how far we have reserved space
virtual Status PreallocateInternal(uint64_t spaceToReserve); virtual IOStatus PreallocateInternal(uint64_t spaceToReserve);
WinWritableImpl(WinFileData* file_data, size_t alignment); WinWritableImpl(WinFileData* file_data, size_t alignment);
@ -304,17 +321,17 @@ class WinWritableImpl {
uint64_t GetAlignement() const { return alignment_; } uint64_t GetAlignement() const { return alignment_; }
Status AppendImpl(const Slice& data); IOStatus AppendImpl(const Slice& data);
// Requires that the data is aligned as specified by // Requires that the data is aligned as specified by
// GetRequiredBufferAlignment() // GetRequiredBufferAlignment()
Status PositionedAppendImpl(const Slice& data, uint64_t offset); IOStatus PositionedAppendImpl(const Slice& data, uint64_t offset);
Status TruncateImpl(uint64_t size); IOStatus TruncateImpl(uint64_t size);
Status CloseImpl(); IOStatus CloseImpl();
Status SyncImpl(); IOStatus SyncImpl(const IOOptions& options, IODebugContext* dbg);
uint64_t GetFileNextWriteOffset() { uint64_t GetFileNextWriteOffset() {
// Double accounting now here with WritableFileWriter // Double accounting now here with WritableFileWriter
@ -326,7 +343,7 @@ class WinWritableImpl {
return next_write_offset_; return next_write_offset_;
} }
Status AllocateImpl(uint64_t offset, uint64_t len); IOStatus AllocateImpl(uint64_t offset, uint64_t len);
public: public:
WinWritableImpl(const WinWritableImpl&) = delete; WinWritableImpl(const WinWritableImpl&) = delete;
@ -335,32 +352,47 @@ class WinWritableImpl {
class WinWritableFile : private WinFileData, class WinWritableFile : private WinFileData,
protected WinWritableImpl, protected WinWritableImpl,
public WritableFile { public FSWritableFile {
public: public:
WinWritableFile(const std::string& fname, HANDLE hFile, size_t alignment, WinWritableFile(const std::string& fname, HANDLE hFile, size_t alignment,
size_t capacity, const EnvOptions& options); size_t capacity, const FileOptions& options);
~WinWritableFile(); ~WinWritableFile();
virtual Status Append(const Slice& data) override; IOStatus Append(const Slice& data, const IOOptions& options,
IODebugContext* dbg) override;
IOStatus Append(const Slice& data, const IOOptions& opts,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) override {
return Append(data, opts, dbg);
}
// Requires that the data is aligned as specified by // Requires that the data is aligned as specified by
// GetRequiredBufferAlignment() // GetRequiredBufferAlignment()
virtual Status PositionedAppend(const Slice& data, uint64_t offset) override; IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& options,
IODebugContext* dbg) override;
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
const IOOptions& opts,
const DataVerificationInfo& /* verification_info */,
IODebugContext* dbg) override {
return PositionedAppend(data, offset, opts, dbg);
}
// Need to implement this so the file is truncated correctly // Need to implement this so the file is truncated correctly
// when buffered and unbuffered mode // when buffered and unbuffered mode
virtual Status Truncate(uint64_t size) override; IOStatus Truncate(uint64_t size, const IOOptions& options,
IODebugContext* dbg) override;
virtual Status Close() override; IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
// write out the cached data to the OS cache // write out the cached data to the OS cache
// This is now taken care of the WritableFileWriter // This is now taken care of the WritableFileWriter
virtual Status Flush() override; IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Sync() override; IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Fsync() override; IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override;
virtual bool IsSyncThreadSafe() const override; virtual bool IsSyncThreadSafe() const override;
@ -370,9 +402,10 @@ class WinWritableFile : private WinFileData,
virtual size_t GetRequiredBufferAlignment() const override; virtual size_t GetRequiredBufferAlignment() const override;
virtual uint64_t GetFileSize() override; uint64_t GetFileSize(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Allocate(uint64_t offset, uint64_t len) override; IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options,
IODebugContext* dbg) override;
virtual size_t GetUniqueId(char* id, size_t max_size) const override; virtual size_t GetUniqueId(char* id, size_t max_size) const override;
}; };
@ -380,10 +413,10 @@ class WinWritableFile : private WinFileData,
class WinRandomRWFile : private WinFileData, class WinRandomRWFile : private WinFileData,
protected WinRandomAccessImpl, protected WinRandomAccessImpl,
protected WinWritableImpl, protected WinWritableImpl,
public RandomRWFile { public FSRandomRWFile {
public: public:
WinRandomRWFile(const std::string& fname, HANDLE hFile, size_t alignment, WinRandomRWFile(const std::string& fname, HANDLE hFile, size_t alignment,
const EnvOptions& options); const FileOptions& options);
~WinRandomRWFile() {} ~WinRandomRWFile() {}
@ -397,45 +430,50 @@ class WinRandomRWFile : private WinFileData,
// Write bytes in `data` at offset `offset`, Returns Status::OK() on success. // Write bytes in `data` at offset `offset`, Returns Status::OK() on success.
// Pass aligned buffer when use_direct_io() returns true. // Pass aligned buffer when use_direct_io() returns true.
virtual Status Write(uint64_t offset, const Slice& data) override; IOStatus Write(uint64_t offset, const Slice& data, const IOOptions& options,
IODebugContext* dbg) override;
// Read up to `n` bytes starting from offset `offset` and store them in // Read up to `n` bytes starting from offset `offset` and store them in
// result, provided `scratch` size should be at least `n`. // result, provided `scratch` size should be at least `n`.
// Returns Status::OK() on success. // Returns Status::OK() on success.
virtual Status Read(uint64_t offset, size_t n, Slice* result, IOStatus Read(uint64_t offset, size_t n, const IOOptions& options,
char* scratch) const override; Slice* result, char* scratch,
IODebugContext* dbg) const override;
virtual Status Flush() override; IOStatus Flush(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Sync() override; IOStatus Sync(const IOOptions& options, IODebugContext* dbg) override;
virtual Status Fsync() override { return Sync(); } IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override {
return Sync(options, dbg);
}
virtual Status Close() override; IOStatus Close(const IOOptions& options, IODebugContext* dbg) override;
}; };
class WinMemoryMappedBuffer : public MemoryMappedFileBuffer { class WinMemoryMappedBuffer : public MemoryMappedFileBuffer {
private: private:
HANDLE file_handle_; HANDLE file_handle_;
HANDLE map_handle_; HANDLE map_handle_;
public:
WinMemoryMappedBuffer(HANDLE file_handle, HANDLE map_handle, void* base, size_t size) : public:
MemoryMappedFileBuffer(base, size), WinMemoryMappedBuffer(HANDLE file_handle, HANDLE map_handle, void* base,
size_t size)
: MemoryMappedFileBuffer(base, size),
file_handle_(file_handle), file_handle_(file_handle),
map_handle_(map_handle) {} map_handle_(map_handle) {}
~WinMemoryMappedBuffer() override; ~WinMemoryMappedBuffer() override;
}; };
class WinDirectory : public Directory { class WinDirectory : public FSDirectory {
HANDLE handle_; HANDLE handle_;
public: public:
explicit WinDirectory(HANDLE h) noexcept : handle_(h) { explicit WinDirectory(HANDLE h) noexcept : handle_(h) {
assert(handle_ != INVALID_HANDLE_VALUE); assert(handle_ != INVALID_HANDLE_VALUE);
} }
~WinDirectory() { ~WinDirectory() { ::CloseHandle(handle_); }
::CloseHandle(handle_); IOStatus Fsync(const IOOptions& options, IODebugContext* dbg) override;
}
virtual Status Fsync() override;
size_t GetUniqueId(char* id, size_t max_size) const override; size_t GetUniqueId(char* id, size_t max_size) const override;
}; };
@ -452,5 +490,5 @@ class WinFileLock : public FileLock {
private: private:
HANDLE hFile_; HANDLE hFile_;
}; };
} } // namespace port
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE

View File

@ -13,31 +13,33 @@
#if defined(OS_WIN) #if defined(OS_WIN)
#include "port/win/win_logger.h" #include "port/win/win_logger.h"
#include "port/win/io_win.h"
#include <algorithm> #include <fcntl.h>
#include <stdio.h> #include <stdio.h>
#include <time.h> #include <time.h>
#include <fcntl.h>
#include <atomic>
#include "rocksdb/env.h" #include <algorithm>
#include <atomic>
#include "monitoring/iostats_context_imp.h" #include "monitoring/iostats_context_imp.h"
#include "port/sys_time.h" #include "port/sys_time.h"
#include "port/win/env_win.h"
#include "port/win/io_win.h"
#include "rocksdb/env.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
namespace port { namespace port {
WinLogger::WinLogger(uint64_t (*gettid)(), Env* env, HANDLE file, WinLogger::WinLogger(uint64_t (*gettid)(),
const std::shared_ptr<WinClock>& clock, HANDLE file,
const InfoLogLevel log_level) const InfoLogLevel log_level)
: Logger(log_level), : Logger(log_level),
file_(file), file_(file),
gettid_(gettid), gettid_(gettid),
log_size_(0), log_size_(0),
last_flush_micros_(0), last_flush_micros_(0),
env_(env), clock_(clock),
flush_pending_(false) { flush_pending_(false) {
assert(file_ != NULL); assert(file_ != NULL);
assert(file_ != INVALID_HANDLE_VALUE); assert(file_ != INVALID_HANDLE_VALUE);
@ -88,7 +90,7 @@ void WinLogger::Flush() {
// for perf reasons. // for perf reasons.
} }
last_flush_micros_ = env_->NowMicros(); last_flush_micros_ = clock_->NowMicros();
} }
void WinLogger::Logv(const char* format, va_list ap) { void WinLogger::Logv(const char* format, va_list ap) {

View File

@ -12,22 +12,23 @@
#pragma once #pragma once
#include <atomic>
#include "rocksdb/env.h"
#include <stdint.h> #include <stdint.h>
#include <windows.h> #include <windows.h>
#include <atomic>
#include <memory>
#include "rocksdb/env.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class Env;
namespace port { namespace port {
class WinClock;
class WinLogger : public ROCKSDB_NAMESPACE::Logger { class WinLogger : public ROCKSDB_NAMESPACE::Logger {
public: public:
WinLogger(uint64_t (*gettid)(), Env* env, HANDLE file, WinLogger(uint64_t (*gettid)(), const std::shared_ptr<WinClock>& clock,
HANDLE file,
const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL); const InfoLogLevel log_level = InfoLogLevel::ERROR_LEVEL);
virtual ~WinLogger(); virtual ~WinLogger();
@ -54,7 +55,7 @@ protected:
uint64_t (*gettid_)(); // Return the thread id for the current thread uint64_t (*gettid_)(); // Return the thread id for the current thread
std::atomic_size_t log_size_; std::atomic_size_t log_size_;
std::atomic_uint_fast64_t last_flush_micros_; std::atomic_uint_fast64_t last_flush_micros_;
Env* env_; std::shared_ptr<WinClock> clock_;
bool flush_pending_; bool flush_pending_;
Status CloseInternal(); Status CloseInternal();

View File

@ -452,6 +452,26 @@ bool IsDirectIOSupported(Env* env, const std::string& dir) {
return s.ok(); return s.ok();
} }
bool IsPrefetchSupported(const std::shared_ptr<FileSystem>& fs,
const std::string& dir) {
bool supported = false;
std::string tmp = TempFileName(dir, 999);
Random rnd(301);
std::string test_string = rnd.RandomString(4096);
Slice data(test_string);
Status s = WriteStringToFile(fs.get(), data, tmp, true);
if (s.ok()) {
std::unique_ptr<FSRandomAccessFile> file;
auto io_s = fs->NewRandomAccessFile(tmp, FileOptions(), &file, nullptr);
if (io_s.ok()) {
supported = !(file->Prefetch(0, data.size(), IOOptions(), nullptr)
.IsNotSupported());
}
s = fs->DeleteFile(tmp, IOOptions(), nullptr);
}
return s.ok() && supported;
}
size_t GetLinesCount(const std::string& fname, const std::string& pattern) { size_t GetLinesCount(const std::string& fname, const std::string& pattern) {
std::stringstream ssbuf; std::stringstream ssbuf;
std::string line; std::string line;

View File

@ -26,6 +26,7 @@
#include "util/mutexlock.h" #include "util/mutexlock.h"
namespace ROCKSDB_NAMESPACE { namespace ROCKSDB_NAMESPACE {
class FileSystem;
class Random; class Random;
class SequentialFile; class SequentialFile;
class SequentialFileReader; class SequentialFileReader;
@ -861,6 +862,9 @@ std::string RandomName(Random* rnd, const size_t len);
bool IsDirectIOSupported(Env* env, const std::string& dir); bool IsDirectIOSupported(Env* env, const std::string& dir);
bool IsPrefetchSupported(const std::shared_ptr<FileSystem>& fs,
const std::string& dir);
// Return the number of lines where a given pattern was found in a file. // Return the number of lines where a given pattern was found in a file.
size_t GetLinesCount(const std::string& fname, const std::string& pattern); size_t GetLinesCount(const std::string& fname, const std::string& pattern);

View File

@ -32,7 +32,6 @@ class DBBenchTest : public testing::Test {
Env::Default()->CreateDir(test_path_); Env::Default()->CreateDir(test_path_);
db_path_ = test_path_ + "/db"; db_path_ = test_path_ + "/db";
wal_path_ = test_path_ + "/wal"; wal_path_ = test_path_ + "/wal";
fs_.reset(new LegacyFileSystemWrapper(Env::Default()));
} }
~DBBenchTest() { ~DBBenchTest() {
@ -114,7 +113,6 @@ class DBBenchTest : public testing::Test {
std::string db_path_; std::string db_path_;
std::string test_path_; std::string test_path_;
std::string wal_path_; std::string wal_path_;
std::unique_ptr<LegacyFileSystemWrapper> fs_;
char arg_buffer_[kArgBufferSize]; char arg_buffer_[kArgBufferSize];
char* argv_[kMaxArgCount]; char* argv_[kMaxArgCount];
@ -130,7 +128,7 @@ TEST_F(DBBenchTest, OptionsFile) {
Options opt = GetDefaultOptions(); Options opt = GetDefaultOptions();
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions()}, kOptionsFileName, {ColumnFamilyOptions()}, kOptionsFileName,
fs_.get())); opt.env->GetFileSystem().get()));
// override the following options as db_bench will not take these // override the following options as db_bench will not take these
// options from the options file // options from the options file
@ -149,7 +147,7 @@ TEST_F(DBBenchTest, OptionsFileUniversal) {
ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"}, ASSERT_OK(PersistRocksDBOptions(DBOptions(opt), {"default"},
{ColumnFamilyOptions(opt)}, kOptionsFileName, {ColumnFamilyOptions(opt)}, kOptionsFileName,
fs_.get())); opt.env->GetFileSystem().get()));
// override the following options as db_bench will not take these // override the following options as db_bench will not take these
// options from the options file // options from the options file

View File

@ -824,10 +824,10 @@ TEST_F(BlobDBTest, SstFileManagerRestart) {
Close(); Close();
// Create 3 dummy trash files under the blob_dir // Create 3 dummy trash files under the blob_dir
LegacyFileSystemWrapper fs(db_options.env); const auto &fs = db_options.env->GetFileSystem();
CreateFile(&fs, blob_dir + "/000666.blob.trash", "", false); CreateFile(fs, blob_dir + "/000666.blob.trash", "", false);
CreateFile(&fs, blob_dir + "/000888.blob.trash", "", true); CreateFile(fs, blob_dir + "/000888.blob.trash", "", true);
CreateFile(&fs, blob_dir + "/something_not_match.trash", "", false); CreateFile(fs, blob_dir + "/something_not_match.trash", "", false);
// Make sure that reopening the DB rescan the existing trash files // Make sure that reopening the DB rescan the existing trash files
Open(bdb_options, db_options); Open(bdb_options, db_options);

View File

@ -7,7 +7,6 @@
#include "rocksdb/utilities/options_util.h" #include "rocksdb/utilities/options_util.h"
#include "env/composite_env_wrapper.h"
#include "file/filename.h" #include "file/filename.h"
#include "options/options_parser.h" #include "options/options_parser.h"
#include "rocksdb/convenience.h" #include "rocksdb/convenience.h"
@ -34,8 +33,8 @@ Status LoadOptionsFromFile(const ConfigOptions& config_options,
std::vector<ColumnFamilyDescriptor>* cf_descs, std::vector<ColumnFamilyDescriptor>* cf_descs,
std::shared_ptr<Cache>* cache) { std::shared_ptr<Cache>* cache) {
RocksDBOptionsParser parser; RocksDBOptionsParser parser;
LegacyFileSystemWrapper fs(config_options.env); const auto& fs = config_options.env->GetFileSystem();
Status s = parser.Parse(config_options, file_name, &fs); Status s = parser.Parse(config_options, file_name, fs.get());
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
@ -149,12 +148,11 @@ Status CheckOptionsCompatibility(
cf_opts.push_back(cf_desc.options); cf_opts.push_back(cf_desc.options);
} }
LegacyFileSystemWrapper fs(config_options.env); const auto& fs = config_options.env->GetFileSystem();
return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile( return RocksDBOptionsParser::VerifyRocksDBOptionsFromFile(
config_options, db_options, cf_names, cf_opts, config_options, db_options, cf_names, cf_opts,
dbpath + "/" + options_file_name, &fs); dbpath + "/" + options_file_name, fs.get());
} }
} // namespace ROCKSDB_NAMESPACE } // namespace ROCKSDB_NAMESPACE