2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2013-10-16 23:59:46 +02:00
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/env.h"
|
Buffer info logs when picking compactions and write them out after releasing the mutex
Summary: Now while the background thread is picking compactions, it writes out multiple info_logs, especially for universal compaction, which introduces a chance of waiting log writing in mutex, which is bad. To remove this risk, write all those info logs to a buffer and flush it after releasing the mutex.
Test Plan:
make all check
check the log lines while running some tests that trigger compactions.
Reviewers: haobo, igor, dhruba
Reviewed By: dhruba
CC: i.am.jin.lei, dhruba, yhchiang, leveldb, nkg-
Differential Revision: https://reviews.facebook.net/D16515
2014-03-04 23:32:55 +01:00
|
|
|
|
2015-06-11 23:18:02 +02:00
|
|
|
#include <thread>
|
2021-01-26 07:07:26 +01:00
|
|
|
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
2019-12-13 23:47:08 +01:00
|
|
|
#include "env/composite_env_wrapper.h"
|
2019-07-09 23:48:07 +02:00
|
|
|
#include "logging/env_logger.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "memory/arena.h"
|
2017-04-13 22:07:33 +02:00
|
|
|
#include "options/db_options.h"
|
2015-07-16 21:10:16 +02:00
|
|
|
#include "port/port.h"
|
2021-06-15 12:42:52 +02:00
|
|
|
#include "rocksdb/convenience.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/options.h"
|
2021-01-26 07:07:26 +01:00
|
|
|
#include "rocksdb/system_clock.h"
|
2019-07-24 02:08:26 +02:00
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
Buffer info logs when picking compactions and write them out after releasing the mutex
Summary: Now while the background thread is picking compactions, it writes out multiple info_logs, especially for universal compaction, which introduces a chance of waiting log writing in mutex, which is bad. To remove this risk, write all those info logs to a buffer and flush it after releasing the mutex.
Test Plan:
make all check
check the log lines while running some tests that trigger compactions.
Reviewers: haobo, igor, dhruba
Reviewed By: dhruba
CC: i.am.jin.lei, dhruba, yhchiang, leveldb, nkg-
Differential Revision: https://reviews.facebook.net/D16515
2014-03-04 23:32:55 +01:00
|
|
|
#include "util/autovector.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2021-01-06 19:48:24 +01:00
|
|
|
namespace {
|
2021-01-26 07:07:26 +01:00
|
|
|
class LegacySystemClock : public SystemClock {
|
|
|
|
private:
|
|
|
|
Env* env_;
|
|
|
|
|
|
|
|
public:
|
|
|
|
explicit LegacySystemClock(Env* env) : env_(env) {}
|
|
|
|
const char* Name() const override { return "Legacy System Clock"; }
|
|
|
|
|
|
|
|
// Returns the number of micro-seconds since some fixed point in time.
|
|
|
|
// It is often used as system time such as in GenericRateLimiter
|
|
|
|
// and other places so a port needs to return system time in order to work.
|
|
|
|
uint64_t NowMicros() override { return env_->NowMicros(); }
|
|
|
|
|
|
|
|
// Returns the number of nano-seconds since some fixed point in time. Only
|
|
|
|
// useful for computing deltas of time in one run.
|
|
|
|
// Default implementation simply relies on NowMicros.
|
|
|
|
// In platform-specific implementations, NowNanos() should return time points
|
|
|
|
// that are MONOTONIC.
|
|
|
|
uint64_t NowNanos() override { return env_->NowNanos(); }
|
|
|
|
|
|
|
|
uint64_t CPUMicros() override { return CPUNanos() / 1000; }
|
|
|
|
uint64_t CPUNanos() override { return env_->NowCPUNanos(); }
|
|
|
|
|
|
|
|
// Sleep/delay the thread for the prescribed number of micro-seconds.
|
|
|
|
void SleepForMicroseconds(int micros) override {
|
|
|
|
env_->SleepForMicroseconds(micros);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the number of seconds since the Epoch, 1970-01-01 00:00:00 (UTC).
|
|
|
|
// Only overwrites *unix_time on success.
|
|
|
|
Status GetCurrentTime(int64_t* unix_time) override {
|
|
|
|
return env_->GetCurrentTime(unix_time);
|
|
|
|
}
|
|
|
|
// Converts seconds-since-Jan-01-1970 to a printable string
|
|
|
|
std::string TimeToString(uint64_t time) override {
|
|
|
|
return env_->TimeToString(time);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2021-01-29 07:08:46 +01:00
|
|
|
class LegacySequentialFileWrapper : public FSSequentialFile {
|
|
|
|
public:
|
|
|
|
explicit LegacySequentialFileWrapper(
|
|
|
|
std::unique_ptr<SequentialFile>&& _target)
|
|
|
|
: target_(std::move(_target)) {}
|
|
|
|
|
|
|
|
IOStatus Read(size_t n, const IOOptions& /*options*/, Slice* result,
|
|
|
|
char* scratch, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Read(n, result, scratch));
|
|
|
|
}
|
|
|
|
IOStatus Skip(uint64_t n) override {
|
|
|
|
return status_to_io_status(target_->Skip(n));
|
|
|
|
}
|
|
|
|
bool use_direct_io() const override { return target_->use_direct_io(); }
|
|
|
|
size_t GetRequiredBufferAlignment() const override {
|
|
|
|
return target_->GetRequiredBufferAlignment();
|
|
|
|
}
|
|
|
|
IOStatus InvalidateCache(size_t offset, size_t length) override {
|
|
|
|
return status_to_io_status(target_->InvalidateCache(offset, length));
|
|
|
|
}
|
|
|
|
IOStatus PositionedRead(uint64_t offset, size_t n,
|
|
|
|
const IOOptions& /*options*/, Slice* result,
|
|
|
|
char* scratch, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(
|
|
|
|
target_->PositionedRead(offset, n, result, scratch));
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<SequentialFile> target_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class LegacyRandomAccessFileWrapper : public FSRandomAccessFile {
|
|
|
|
public:
|
|
|
|
explicit LegacyRandomAccessFileWrapper(
|
|
|
|
std::unique_ptr<RandomAccessFile>&& target)
|
|
|
|
: target_(std::move(target)) {}
|
|
|
|
|
|
|
|
IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/,
|
|
|
|
Slice* result, char* scratch,
|
|
|
|
IODebugContext* /*dbg*/) const override {
|
|
|
|
return status_to_io_status(target_->Read(offset, n, result, scratch));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus MultiRead(FSReadRequest* fs_reqs, size_t num_reqs,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::vector<ReadRequest> reqs;
|
|
|
|
Status status;
|
|
|
|
|
|
|
|
reqs.reserve(num_reqs);
|
|
|
|
for (size_t i = 0; i < num_reqs; ++i) {
|
|
|
|
ReadRequest req;
|
|
|
|
|
|
|
|
req.offset = fs_reqs[i].offset;
|
|
|
|
req.len = fs_reqs[i].len;
|
|
|
|
req.scratch = fs_reqs[i].scratch;
|
|
|
|
req.status = Status::OK();
|
|
|
|
|
|
|
|
reqs.emplace_back(req);
|
|
|
|
}
|
|
|
|
status = target_->MultiRead(reqs.data(), num_reqs);
|
|
|
|
for (size_t i = 0; i < num_reqs; ++i) {
|
|
|
|
fs_reqs[i].result = reqs[i].result;
|
|
|
|
fs_reqs[i].status = status_to_io_status(std::move(reqs[i].status));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(status));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Prefetch(offset, n));
|
|
|
|
}
|
|
|
|
size_t GetUniqueId(char* id, size_t max_size) const override {
|
|
|
|
return target_->GetUniqueId(id, max_size);
|
|
|
|
}
|
|
|
|
void Hint(AccessPattern pattern) override {
|
|
|
|
target_->Hint((RandomAccessFile::AccessPattern)pattern);
|
|
|
|
}
|
|
|
|
bool use_direct_io() const override { return target_->use_direct_io(); }
|
|
|
|
size_t GetRequiredBufferAlignment() const override {
|
|
|
|
return target_->GetRequiredBufferAlignment();
|
|
|
|
}
|
|
|
|
IOStatus InvalidateCache(size_t offset, size_t length) override {
|
|
|
|
return status_to_io_status(target_->InvalidateCache(offset, length));
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<RandomAccessFile> target_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class LegacyRandomRWFileWrapper : public FSRandomRWFile {
|
|
|
|
public:
|
|
|
|
explicit LegacyRandomRWFileWrapper(std::unique_ptr<RandomRWFile>&& target)
|
|
|
|
: target_(std::move(target)) {}
|
|
|
|
|
|
|
|
bool use_direct_io() const override { return target_->use_direct_io(); }
|
|
|
|
size_t GetRequiredBufferAlignment() const override {
|
|
|
|
return target_->GetRequiredBufferAlignment();
|
|
|
|
}
|
|
|
|
IOStatus Write(uint64_t offset, const Slice& data,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Write(offset, data));
|
|
|
|
}
|
|
|
|
IOStatus Read(uint64_t offset, size_t n, const IOOptions& /*options*/,
|
|
|
|
Slice* result, char* scratch,
|
|
|
|
IODebugContext* /*dbg*/) const override {
|
|
|
|
return status_to_io_status(target_->Read(offset, n, result, scratch));
|
|
|
|
}
|
|
|
|
IOStatus Flush(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Flush());
|
|
|
|
}
|
|
|
|
IOStatus Sync(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Sync());
|
|
|
|
}
|
|
|
|
IOStatus Fsync(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Fsync());
|
|
|
|
}
|
|
|
|
IOStatus Close(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Close());
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<RandomRWFile> target_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class LegacyWritableFileWrapper : public FSWritableFile {
|
|
|
|
public:
|
|
|
|
explicit LegacyWritableFileWrapper(std::unique_ptr<WritableFile>&& _target)
|
|
|
|
: target_(std::move(_target)) {}
|
|
|
|
|
|
|
|
IOStatus Append(const Slice& data, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Append(data));
|
|
|
|
}
|
|
|
|
IOStatus Append(const Slice& data, const IOOptions& /*options*/,
|
|
|
|
const DataVerificationInfo& /*verification_info*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Append(data));
|
|
|
|
}
|
|
|
|
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->PositionedAppend(data, offset));
|
|
|
|
}
|
|
|
|
IOStatus PositionedAppend(const Slice& data, uint64_t offset,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
const DataVerificationInfo& /*verification_info*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->PositionedAppend(data, offset));
|
|
|
|
}
|
|
|
|
IOStatus Truncate(uint64_t size, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Truncate(size));
|
|
|
|
}
|
|
|
|
IOStatus Close(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Close());
|
|
|
|
}
|
|
|
|
IOStatus Flush(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Flush());
|
|
|
|
}
|
|
|
|
IOStatus Sync(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Sync());
|
|
|
|
}
|
|
|
|
IOStatus Fsync(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Fsync());
|
|
|
|
}
|
|
|
|
bool IsSyncThreadSafe() const override { return target_->IsSyncThreadSafe(); }
|
|
|
|
|
|
|
|
bool use_direct_io() const override { return target_->use_direct_io(); }
|
|
|
|
|
|
|
|
size_t GetRequiredBufferAlignment() const override {
|
|
|
|
return target_->GetRequiredBufferAlignment();
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override {
|
|
|
|
target_->SetWriteLifeTimeHint(hint);
|
|
|
|
}
|
|
|
|
|
|
|
|
Env::WriteLifeTimeHint GetWriteLifeTimeHint() override {
|
|
|
|
return target_->GetWriteLifeTimeHint();
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t GetFileSize(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return target_->GetFileSize();
|
|
|
|
}
|
|
|
|
|
|
|
|
void SetPreallocationBlockSize(size_t size) override {
|
|
|
|
target_->SetPreallocationBlockSize(size);
|
|
|
|
}
|
|
|
|
|
|
|
|
void GetPreallocationStatus(size_t* block_size,
|
|
|
|
size_t* last_allocated_block) override {
|
|
|
|
target_->GetPreallocationStatus(block_size, last_allocated_block);
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t GetUniqueId(char* id, size_t max_size) const override {
|
|
|
|
return target_->GetUniqueId(id, max_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus InvalidateCache(size_t offset, size_t length) override {
|
|
|
|
return status_to_io_status(target_->InvalidateCache(offset, length));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus RangeSync(uint64_t offset, uint64_t nbytes,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->RangeSync(offset, nbytes));
|
|
|
|
}
|
|
|
|
|
|
|
|
void PrepareWrite(size_t offset, size_t len, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
target_->PrepareWrite(offset, len);
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Allocate(offset, len));
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<WritableFile> target_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class LegacyDirectoryWrapper : public FSDirectory {
|
|
|
|
public:
|
|
|
|
explicit LegacyDirectoryWrapper(std::unique_ptr<Directory>&& target)
|
|
|
|
: target_(std::move(target)) {}
|
|
|
|
|
|
|
|
IOStatus Fsync(const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Fsync());
|
|
|
|
}
|
|
|
|
size_t GetUniqueId(char* id, size_t max_size) const override {
|
|
|
|
return target_->GetUniqueId(id, max_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<Directory> target_;
|
|
|
|
};
|
|
|
|
|
2021-01-06 19:48:24 +01:00
|
|
|
class LegacyFileSystemWrapper : public FileSystem {
|
|
|
|
public:
|
|
|
|
// Initialize an EnvWrapper that delegates all calls to *t
|
|
|
|
explicit LegacyFileSystemWrapper(Env* t) : target_(t) {}
|
|
|
|
~LegacyFileSystemWrapper() override {}
|
|
|
|
|
|
|
|
const char* Name() const override { return "Legacy File System"; }
|
|
|
|
|
|
|
|
// Return the target to which this Env forwards all calls
|
|
|
|
Env* target() const { return target_; }
|
|
|
|
|
|
|
|
// The following text is boilerplate that forwards all methods to target()
|
|
|
|
IOStatus NewSequentialFile(const std::string& f, const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSSequentialFile>* r,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<SequentialFile> file;
|
|
|
|
Status s = target_->NewSequentialFile(f, &file, file_opts);
|
|
|
|
if (s.ok()) {
|
|
|
|
r->reset(new LegacySequentialFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus NewRandomAccessFile(const std::string& f,
|
|
|
|
const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSRandomAccessFile>* r,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<RandomAccessFile> file;
|
|
|
|
Status s = target_->NewRandomAccessFile(f, &file, file_opts);
|
|
|
|
if (s.ok()) {
|
|
|
|
r->reset(new LegacyRandomAccessFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus NewWritableFile(const std::string& f, const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSWritableFile>* r,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<WritableFile> file;
|
|
|
|
Status s = target_->NewWritableFile(f, &file, file_opts);
|
|
|
|
if (s.ok()) {
|
|
|
|
r->reset(new LegacyWritableFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus ReopenWritableFile(const std::string& fname,
|
|
|
|
const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSWritableFile>* result,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<WritableFile> file;
|
|
|
|
Status s = target_->ReopenWritableFile(fname, &file, file_opts);
|
|
|
|
if (s.ok()) {
|
|
|
|
result->reset(new LegacyWritableFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus ReuseWritableFile(const std::string& fname,
|
|
|
|
const std::string& old_fname,
|
|
|
|
const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSWritableFile>* r,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<WritableFile> file;
|
|
|
|
Status s = target_->ReuseWritableFile(fname, old_fname, &file, file_opts);
|
|
|
|
if (s.ok()) {
|
|
|
|
r->reset(new LegacyWritableFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus NewRandomRWFile(const std::string& fname,
|
|
|
|
const FileOptions& file_opts,
|
|
|
|
std::unique_ptr<FSRandomRWFile>* result,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<RandomRWFile> file;
|
|
|
|
Status s = target_->NewRandomRWFile(fname, &file, file_opts);
|
|
|
|
if (s.ok()) {
|
|
|
|
result->reset(new LegacyRandomRWFileWrapper(std::move(file)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus NewMemoryMappedFileBuffer(
|
|
|
|
const std::string& fname,
|
|
|
|
std::unique_ptr<MemoryMappedFileBuffer>* result) override {
|
|
|
|
return status_to_io_status(
|
|
|
|
target_->NewMemoryMappedFileBuffer(fname, result));
|
|
|
|
}
|
|
|
|
IOStatus NewDirectory(const std::string& name, const IOOptions& /*io_opts*/,
|
|
|
|
std::unique_ptr<FSDirectory>* result,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
std::unique_ptr<Directory> dir;
|
|
|
|
Status s = target_->NewDirectory(name, &dir);
|
|
|
|
if (s.ok()) {
|
|
|
|
result->reset(new LegacyDirectoryWrapper(std::move(dir)));
|
|
|
|
}
|
|
|
|
return status_to_io_status(std::move(s));
|
|
|
|
}
|
|
|
|
IOStatus FileExists(const std::string& f, const IOOptions& /*io_opts*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->FileExists(f));
|
|
|
|
}
|
|
|
|
IOStatus GetChildren(const std::string& dir, const IOOptions& /*io_opts*/,
|
|
|
|
std::vector<std::string>* r,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->GetChildren(dir, r));
|
|
|
|
}
|
|
|
|
IOStatus GetChildrenFileAttributes(const std::string& dir,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
std::vector<FileAttributes>* result,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->GetChildrenFileAttributes(dir, result));
|
|
|
|
}
|
|
|
|
IOStatus DeleteFile(const std::string& f, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->DeleteFile(f));
|
|
|
|
}
|
|
|
|
IOStatus Truncate(const std::string& fname, size_t size,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->Truncate(fname, size));
|
|
|
|
}
|
|
|
|
IOStatus CreateDir(const std::string& d, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->CreateDir(d));
|
|
|
|
}
|
|
|
|
IOStatus CreateDirIfMissing(const std::string& d,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->CreateDirIfMissing(d));
|
|
|
|
}
|
|
|
|
IOStatus DeleteDir(const std::string& d, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->DeleteDir(d));
|
|
|
|
}
|
|
|
|
IOStatus GetFileSize(const std::string& f, const IOOptions& /*options*/,
|
|
|
|
uint64_t* s, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->GetFileSize(f, s));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus GetFileModificationTime(const std::string& fname,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
uint64_t* file_mtime,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(
|
|
|
|
target_->GetFileModificationTime(fname, file_mtime));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus GetAbsolutePath(const std::string& db_path,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
std::string* output_path,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->GetAbsolutePath(db_path, output_path));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus RenameFile(const std::string& s, const std::string& t,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->RenameFile(s, t));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus LinkFile(const std::string& s, const std::string& t,
|
|
|
|
const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->LinkFile(s, t));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*options*/,
|
|
|
|
uint64_t* count, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->NumFileLinks(fname, count));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus AreFilesSame(const std::string& first, const std::string& second,
|
|
|
|
const IOOptions& /*options*/, bool* res,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->AreFilesSame(first, second, res));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus LockFile(const std::string& f, const IOOptions& /*options*/,
|
|
|
|
FileLock** l, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->LockFile(f, l));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus UnlockFile(FileLock* l, const IOOptions& /*options*/,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->UnlockFile(l));
|
|
|
|
}
|
|
|
|
|
|
|
|
IOStatus GetTestDirectory(const IOOptions& /*options*/, std::string* path,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->GetTestDirectory(path));
|
|
|
|
}
|
|
|
|
IOStatus NewLogger(const std::string& fname, const IOOptions& /*options*/,
|
|
|
|
std::shared_ptr<Logger>* result,
|
|
|
|
IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->NewLogger(fname, result));
|
|
|
|
}
|
|
|
|
|
|
|
|
void SanitizeFileOptions(FileOptions* opts) const override {
|
|
|
|
target_->SanitizeEnvOptions(opts);
|
|
|
|
}
|
|
|
|
|
|
|
|
FileOptions OptimizeForLogRead(
|
|
|
|
const FileOptions& file_options) const override {
|
|
|
|
return target_->OptimizeForLogRead(file_options);
|
|
|
|
}
|
|
|
|
FileOptions OptimizeForManifestRead(
|
|
|
|
const FileOptions& file_options) const override {
|
|
|
|
return target_->OptimizeForManifestRead(file_options);
|
|
|
|
}
|
|
|
|
FileOptions OptimizeForLogWrite(const FileOptions& file_options,
|
|
|
|
const DBOptions& db_options) const override {
|
|
|
|
return target_->OptimizeForLogWrite(file_options, db_options);
|
|
|
|
}
|
|
|
|
FileOptions OptimizeForManifestWrite(
|
|
|
|
const FileOptions& file_options) const override {
|
|
|
|
return target_->OptimizeForManifestWrite(file_options);
|
|
|
|
}
|
|
|
|
FileOptions OptimizeForCompactionTableWrite(
|
|
|
|
const FileOptions& file_options,
|
|
|
|
const ImmutableDBOptions& immutable_ops) const override {
|
|
|
|
return target_->OptimizeForCompactionTableWrite(file_options,
|
|
|
|
immutable_ops);
|
|
|
|
}
|
|
|
|
FileOptions OptimizeForCompactionTableRead(
|
|
|
|
const FileOptions& file_options,
|
|
|
|
const ImmutableDBOptions& db_options) const override {
|
|
|
|
return target_->OptimizeForCompactionTableRead(file_options, db_options);
|
|
|
|
}
|
2021-04-07 22:37:36 +02:00
|
|
|
FileOptions OptimizeForBlobFileRead(
|
|
|
|
const FileOptions& file_options,
|
|
|
|
const ImmutableDBOptions& db_options) const override {
|
|
|
|
return target_->OptimizeForBlobFileRead(file_options, db_options);
|
|
|
|
}
|
2021-01-06 19:48:24 +01:00
|
|
|
|
|
|
|
#ifdef GetFreeSpace
|
|
|
|
#undef GetFreeSpace
|
|
|
|
#endif
|
|
|
|
IOStatus GetFreeSpace(const std::string& path, const IOOptions& /*options*/,
|
|
|
|
uint64_t* diskfree, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->GetFreeSpace(path, diskfree));
|
|
|
|
}
|
|
|
|
IOStatus IsDirectory(const std::string& path, const IOOptions& /*options*/,
|
|
|
|
bool* is_dir, IODebugContext* /*dbg*/) override {
|
|
|
|
return status_to_io_status(target_->IsDirectory(path, is_dir));
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
Env* target_;
|
|
|
|
};
|
|
|
|
} // end anonymous namespace
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2020-03-24 05:50:42 +01:00
|
|
|
Env::Env() : thread_status_updater_(nullptr) {
|
|
|
|
file_system_ = std::make_shared<LegacyFileSystemWrapper>(this);
|
2021-01-26 07:07:26 +01:00
|
|
|
system_clock_ = std::make_shared<LegacySystemClock>(this);
|
2020-03-24 05:50:42 +01:00
|
|
|
}
|
|
|
|
|
2021-01-26 07:07:26 +01:00
|
|
|
Env::Env(const std::shared_ptr<FileSystem>& fs)
|
|
|
|
: thread_status_updater_(nullptr), file_system_(fs) {
|
|
|
|
system_clock_ = std::make_shared<LegacySystemClock>(this);
|
|
|
|
}
|
|
|
|
|
|
|
|
Env::Env(const std::shared_ptr<FileSystem>& fs,
|
|
|
|
const std::shared_ptr<SystemClock>& clock)
|
|
|
|
: thread_status_updater_(nullptr), file_system_(fs), system_clock_(clock) {}
|
2020-03-24 05:50:42 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Env::~Env() {
|
|
|
|
}
|
|
|
|
|
2019-07-09 23:48:07 +02:00
|
|
|
Status Env::NewLogger(const std::string& fname,
|
|
|
|
std::shared_ptr<Logger>* result) {
|
|
|
|
return NewEnvLogger(fname, this, result);
|
|
|
|
}
|
|
|
|
|
2019-07-24 02:08:26 +02:00
|
|
|
Status Env::LoadEnv(const std::string& value, Env** result) {
|
2021-06-15 12:42:52 +02:00
|
|
|
return CreateFromString(ConfigOptions(), value, result);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status Env::CreateFromString(const ConfigOptions& config_options,
|
|
|
|
const std::string& value, Env** result) {
|
2019-07-24 02:08:26 +02:00
|
|
|
Env* env = *result;
|
|
|
|
Status s;
|
|
|
|
#ifndef ROCKSDB_LITE
|
2021-06-15 12:42:52 +02:00
|
|
|
(void)config_options;
|
2019-07-24 02:08:26 +02:00
|
|
|
s = ObjectRegistry::NewInstance()->NewStaticObject<Env>(value, &env);
|
|
|
|
#else
|
2021-06-15 12:42:52 +02:00
|
|
|
(void)config_options;
|
2020-08-07 00:16:50 +02:00
|
|
|
s = Status::NotSupported("Cannot load environment in LITE mode", value);
|
2019-07-24 02:08:26 +02:00
|
|
|
#endif
|
|
|
|
if (s.ok()) {
|
|
|
|
*result = env;
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2019-10-09 04:17:39 +02:00
|
|
|
Status Env::LoadEnv(const std::string& value, Env** result,
|
|
|
|
std::shared_ptr<Env>* guard) {
|
2021-06-15 12:42:52 +02:00
|
|
|
return CreateFromString(ConfigOptions(), value, result, guard);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status Env::CreateFromString(const ConfigOptions& config_options,
|
|
|
|
const std::string& value, Env** result,
|
|
|
|
std::shared_ptr<Env>* guard) {
|
2019-10-09 04:17:39 +02:00
|
|
|
assert(result);
|
2021-06-15 12:42:52 +02:00
|
|
|
if (value.empty()) {
|
|
|
|
*result = Env::Default();
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2019-10-09 04:17:39 +02:00
|
|
|
Status s;
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
Env* env = nullptr;
|
|
|
|
std::unique_ptr<Env> uniq_guard;
|
|
|
|
std::string err_msg;
|
|
|
|
assert(guard != nullptr);
|
2021-06-15 12:42:52 +02:00
|
|
|
(void)config_options;
|
2019-10-09 04:17:39 +02:00
|
|
|
env = ObjectRegistry::NewInstance()->NewObject<Env>(value, &uniq_guard,
|
|
|
|
&err_msg);
|
|
|
|
if (!env) {
|
2021-06-15 12:42:52 +02:00
|
|
|
s = Status::NotSupported(std::string("Cannot load ") + Env::Type() + ": " +
|
|
|
|
value);
|
2019-10-09 04:17:39 +02:00
|
|
|
env = Env::Default();
|
|
|
|
}
|
|
|
|
if (s.ok() && uniq_guard) {
|
|
|
|
guard->reset(uniq_guard.release());
|
|
|
|
*result = guard->get();
|
|
|
|
} else {
|
|
|
|
*result = env;
|
|
|
|
}
|
|
|
|
#else
|
2021-06-15 12:42:52 +02:00
|
|
|
(void)config_options;
|
2019-10-09 04:17:39 +02:00
|
|
|
(void)result;
|
|
|
|
(void)guard;
|
2020-08-07 00:16:50 +02:00
|
|
|
s = Status::NotSupported("Cannot load environment in LITE mode", value);
|
2019-10-09 04:17:39 +02:00
|
|
|
#endif
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2021-06-15 12:42:52 +02:00
|
|
|
Status Env::CreateFromUri(const ConfigOptions& config_options,
|
|
|
|
const std::string& env_uri, const std::string& fs_uri,
|
|
|
|
Env** result, std::shared_ptr<Env>* guard) {
|
|
|
|
*result = config_options.env;
|
|
|
|
if (env_uri.empty() && fs_uri.empty()) {
|
|
|
|
// Neither specified. Use the default
|
|
|
|
guard->reset();
|
|
|
|
return Status::OK();
|
|
|
|
} else if (!env_uri.empty() && !fs_uri.empty()) {
|
|
|
|
// Both specified. Cannot choose. Return Invalid
|
|
|
|
return Status::InvalidArgument("cannot specify both fs_uri and env_uri");
|
|
|
|
} else if (fs_uri.empty()) { // Only have an ENV URI. Create an Env from it
|
|
|
|
return CreateFromString(config_options, env_uri, result, guard);
|
|
|
|
} else {
|
|
|
|
std::shared_ptr<FileSystem> fs;
|
|
|
|
Status s = FileSystem::CreateFromString(config_options, fs_uri, &fs);
|
|
|
|
if (s.ok()) {
|
|
|
|
guard->reset(new CompositeEnvWrapper(*result, fs));
|
|
|
|
*result = guard->get();
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-19 02:25:37 +02:00
|
|
|
std::string Env::PriorityToString(Env::Priority priority) {
|
|
|
|
switch (priority) {
|
|
|
|
case Env::Priority::BOTTOM:
|
|
|
|
return "Bottom";
|
|
|
|
case Env::Priority::LOW:
|
|
|
|
return "Low";
|
|
|
|
case Env::Priority::HIGH:
|
|
|
|
return "High";
|
2019-03-20 01:24:09 +01:00
|
|
|
case Env::Priority::USER:
|
|
|
|
return "User";
|
2018-04-19 02:25:37 +02:00
|
|
|
case Env::Priority::TOTAL:
|
|
|
|
assert(false);
|
|
|
|
}
|
|
|
|
return "Invalid";
|
|
|
|
}
|
|
|
|
|
2015-06-11 23:18:02 +02:00
|
|
|
uint64_t Env::GetThreadID() const {
|
|
|
|
std::hash<std::thread::id> hasher;
|
|
|
|
return hasher(std::this_thread::get_id());
|
|
|
|
}
|
|
|
|
|
2015-10-08 04:11:09 +02:00
|
|
|
Status Env::ReuseWritableFile(const std::string& fname,
|
|
|
|
const std::string& old_fname,
|
2018-11-09 20:17:34 +01:00
|
|
|
std::unique_ptr<WritableFile>* result,
|
2015-10-08 04:11:09 +02:00
|
|
|
const EnvOptions& options) {
|
|
|
|
Status s = RenameFile(old_fname, fname);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
return NewWritableFile(fname, result, options);
|
|
|
|
}
|
|
|
|
|
2016-02-09 23:54:32 +01:00
|
|
|
Status Env::GetChildrenFileAttributes(const std::string& dir,
|
|
|
|
std::vector<FileAttributes>* result) {
|
|
|
|
assert(result != nullptr);
|
|
|
|
std::vector<std::string> child_fnames;
|
|
|
|
Status s = GetChildren(dir, &child_fnames);
|
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
result->resize(child_fnames.size());
|
|
|
|
size_t result_size = 0;
|
|
|
|
for (size_t i = 0; i < child_fnames.size(); ++i) {
|
|
|
|
const std::string path = dir + "/" + child_fnames[i];
|
|
|
|
if (!(s = GetFileSize(path, &(*result)[result_size].size_bytes)).ok()) {
|
|
|
|
if (FileExists(path).IsNotFound()) {
|
|
|
|
// The file may have been deleted since we listed the directory
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
(*result)[result_size].name = std::move(child_fnames[i]);
|
|
|
|
result_size++;
|
|
|
|
}
|
|
|
|
result->resize(result_size);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2020-10-19 20:37:05 +02:00
|
|
|
Status Env::GetHostNameString(std::string* result) {
|
|
|
|
std::array<char, kMaxHostNameLen> hostname_buf;
|
|
|
|
Status s = GetHostName(hostname_buf.data(), hostname_buf.size());
|
|
|
|
if (s.ok()) {
|
|
|
|
hostname_buf[hostname_buf.size() - 1] = '\0';
|
|
|
|
result->assign(hostname_buf.data());
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
SequentialFile::~SequentialFile() {
|
|
|
|
}
|
|
|
|
|
|
|
|
RandomAccessFile::~RandomAccessFile() {
|
|
|
|
}
|
|
|
|
|
|
|
|
WritableFile::~WritableFile() {
|
|
|
|
}
|
|
|
|
|
2018-04-30 21:23:45 +02:00
|
|
|
MemoryMappedFileBuffer::~MemoryMappedFileBuffer() {}
|
|
|
|
|
2018-03-07 01:13:05 +01:00
|
|
|
Logger::~Logger() {}
|
2018-01-16 19:57:56 +01:00
|
|
|
|
|
|
|
Status Logger::Close() {
|
|
|
|
if (!closed_) {
|
|
|
|
closed_ = true;
|
|
|
|
return CloseImpl();
|
2018-02-23 22:50:02 +01:00
|
|
|
} else {
|
|
|
|
return Status::OK();
|
2018-01-16 19:57:56 +01:00
|
|
|
}
|
2011-07-21 04:40:18 +02:00
|
|
|
}
|
|
|
|
|
2018-02-23 22:50:02 +01:00
|
|
|
Status Logger::CloseImpl() { return Status::NotSupported(); }
|
2018-01-16 19:57:56 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
FileLock::~FileLock() {
|
|
|
|
}
|
|
|
|
|
2013-11-07 20:31:56 +01:00
|
|
|
void LogFlush(Logger *info_log) {
|
|
|
|
if (info_log) {
|
|
|
|
info_log->Flush();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
static void Logv(Logger *info_log, const char* format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) {
|
2014-04-11 00:27:42 +02:00
|
|
|
info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap);
|
2013-01-20 11:07:13 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
void Log(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Logv(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
2015-07-16 21:10:16 +02:00
|
|
|
void Logger::Logv(const InfoLogLevel log_level, const char* format, va_list ap) {
|
|
|
|
static const char* kInfoLogLevelNames[5] = { "DEBUG", "INFO", "WARN",
|
|
|
|
"ERROR", "FATAL" };
|
|
|
|
if (log_level < log_level_) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (log_level == InfoLogLevel::INFO_LEVEL) {
|
|
|
|
// Doesn't print log level if it is INFO level.
|
|
|
|
// This is to avoid unexpected performance regression after we add
|
|
|
|
// the feature of log level. All the logs before we add the feature
|
|
|
|
// are INFO level. We don't want to add extra costs to those existing
|
|
|
|
// logging.
|
|
|
|
Logv(format, ap);
|
2019-02-16 01:56:58 +01:00
|
|
|
} else if (log_level == InfoLogLevel::HEADER_LEVEL) {
|
|
|
|
LogHeader(format, ap);
|
2015-07-16 21:10:16 +02:00
|
|
|
} else {
|
|
|
|
char new_format[500];
|
|
|
|
snprintf(new_format, sizeof(new_format) - 1, "[%s] %s",
|
|
|
|
kInfoLogLevelNames[log_level], format);
|
|
|
|
Logv(new_format, ap);
|
|
|
|
}
|
2020-09-30 01:04:52 +02:00
|
|
|
|
|
|
|
if (log_level >= InfoLogLevel::WARN_LEVEL &&
|
|
|
|
log_level != InfoLogLevel::HEADER_LEVEL) {
|
|
|
|
// Log messages with severity of warning or higher should be rare and are
|
|
|
|
// sometimes followed by an unclean crash. We want to be sure important
|
|
|
|
// messages are not lost in an application buffer when that happens.
|
|
|
|
Flush();
|
|
|
|
}
|
2015-07-16 21:10:16 +02:00
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
static void Logv(const InfoLogLevel log_level, Logger *info_log, const char *format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= log_level) {
|
2015-07-03 02:14:39 +02:00
|
|
|
if (log_level == InfoLogLevel::HEADER_LEVEL) {
|
|
|
|
info_log->LogHeader(format, ap);
|
|
|
|
} else {
|
|
|
|
info_log->Logv(log_level, format, ap);
|
|
|
|
}
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
void Log(const InfoLogLevel log_level, Logger* info_log, const char* format,
|
|
|
|
...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Logv(log_level, info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void Headerv(Logger *info_log, const char *format, va_list ap) {
|
2015-02-02 18:47:24 +01:00
|
|
|
if (info_log) {
|
|
|
|
info_log->LogHeader(format, ap);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
void Header(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Headerv(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void Debugv(Logger* info_log, const char* format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::DEBUG_LEVEL) {
|
2014-04-11 00:27:42 +02:00
|
|
|
info_log->Logv(InfoLogLevel::DEBUG_LEVEL, format, ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
void Debug(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Debugv(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void Infov(Logger* info_log, const char* format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::INFO_LEVEL) {
|
2014-04-11 00:27:42 +02:00
|
|
|
info_log->Logv(InfoLogLevel::INFO_LEVEL, format, ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
void Info(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Infov(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void Warnv(Logger* info_log, const char* format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::WARN_LEVEL) {
|
2014-04-11 00:27:42 +02:00
|
|
|
info_log->Logv(InfoLogLevel::WARN_LEVEL, format, ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
}
|
2018-04-13 19:58:25 +02:00
|
|
|
|
|
|
|
void Warn(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Warnv(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void Errorv(Logger* info_log, const char* format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::ERROR_LEVEL) {
|
2014-04-11 00:27:42 +02:00
|
|
|
info_log->Logv(InfoLogLevel::ERROR_LEVEL, format, ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
}
|
2018-04-13 19:58:25 +02:00
|
|
|
|
|
|
|
void Error(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Errorv(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void Fatalv(Logger* info_log, const char* format, va_list ap) {
|
2014-10-30 21:36:18 +01:00
|
|
|
if (info_log && info_log->GetInfoLogLevel() <= InfoLogLevel::FATAL_LEVEL) {
|
2014-04-11 00:27:42 +02:00
|
|
|
info_log->Logv(InfoLogLevel::FATAL_LEVEL, format, ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-13 19:58:25 +02:00
|
|
|
void Fatal(Logger* info_log, const char* format, ...) {
|
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Fatalv(info_log, format, ap);
|
|
|
|
va_end(ap);
|
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void LogFlush(const std::shared_ptr<Logger>& info_log) {
|
2018-04-13 19:58:25 +02:00
|
|
|
LogFlush(info_log.get());
|
2013-11-07 20:31:56 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Log(const InfoLogLevel log_level, const std::shared_ptr<Logger>& info_log,
|
2014-02-26 23:41:28 +01:00
|
|
|
const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Logv(log_level, info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Header(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Headerv(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2015-02-02 18:47:24 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Debug(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Debugv(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Info(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Infov(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Warn(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Warnv(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Error(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Errorv(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Fatal(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Fatalv(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2014-02-26 23:41:28 +01:00
|
|
|
}
|
|
|
|
|
2018-11-09 20:17:34 +01:00
|
|
|
void Log(const std::shared_ptr<Logger>& info_log, const char* format, ...) {
|
2018-04-13 19:58:25 +02:00
|
|
|
va_list ap;
|
|
|
|
va_start(ap, format);
|
|
|
|
Logv(info_log.get(), format, ap);
|
|
|
|
va_end(ap);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
Status WriteStringToFile(Env* env, const Slice& data, const std::string& fname,
|
|
|
|
bool should_sync) {
|
2021-01-06 19:48:24 +01:00
|
|
|
const auto& fs = env->GetFileSystem();
|
|
|
|
return WriteStringToFile(fs.get(), data, fname, should_sync);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status ReadFileToString(Env* env, const std::string& fname, std::string* data) {
|
2021-01-06 19:48:24 +01:00
|
|
|
const auto& fs = env->GetFileSystem();
|
|
|
|
return ReadFileToString(fs.get(), fname, data);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
EnvWrapper::~EnvWrapper() {
|
|
|
|
}
|
|
|
|
|
2013-06-08 00:35:17 +02:00
|
|
|
namespace { // anonymous namespace
|
|
|
|
|
2014-02-05 22:12:23 +01:00
|
|
|
void AssignEnvOptions(EnvOptions* env_options, const DBOptions& options) {
|
2013-06-08 00:35:17 +02:00
|
|
|
env_options->use_mmap_reads = options.allow_mmap_reads;
|
|
|
|
env_options->use_mmap_writes = options.allow_mmap_writes;
|
2016-10-28 19:36:05 +02:00
|
|
|
env_options->use_direct_reads = options.use_direct_reads;
|
2013-06-08 00:35:17 +02:00
|
|
|
env_options->set_fd_cloexec = options.is_fd_close_on_exec;
|
2013-06-14 07:49:46 +02:00
|
|
|
env_options->bytes_per_sync = options.bytes_per_sync;
|
2015-10-27 22:44:16 +01:00
|
|
|
env_options->compaction_readahead_size = options.compaction_readahead_size;
|
2015-10-29 23:52:32 +01:00
|
|
|
env_options->random_access_max_buffer_size =
|
|
|
|
options.random_access_max_buffer_size;
|
2014-07-08 21:31:49 +02:00
|
|
|
env_options->rate_limiter = options.rate_limiter.get();
|
2015-11-16 21:56:21 +01:00
|
|
|
env_options->writable_file_max_buffer_size =
|
|
|
|
options.writable_file_max_buffer_size;
|
2015-10-07 19:04:05 +02:00
|
|
|
env_options->allow_fallocate = options.allow_fallocate;
|
Optionally wait on bytes_per_sync to smooth I/O (#5183)
Summary:
The existing implementation does not guarantee bytes reach disk every `bytes_per_sync` when writing SST files, or every `wal_bytes_per_sync` when writing WALs. This can cause confusing behavior for users who enable this feature to avoid large syncs during flush and compaction, but then end up hitting them anyways.
My understanding of the existing behavior is we used `sync_file_range` with `SYNC_FILE_RANGE_WRITE` to submit ranges for async writeback, such that we could continue processing the next range of bytes while that I/O is happening. I believe we can preserve that benefit while also limiting how far the processing can get ahead of the I/O, which prevents huge syncs from happening when the file finishes.
Consider this `sync_file_range` usage: `sync_file_range(fd_, 0, static_cast<off_t>(offset + nbytes), SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE)`. Expanding the range to start at 0 and adding the `SYNC_FILE_RANGE_WAIT_BEFORE` flag causes any pending writeback (like from a previous call to `sync_file_range`) to finish before it proceeds to submit the latest `nbytes` for writeback. The latest `nbytes` are still written back asynchronously, unless processing exceeds I/O speed, in which case the following `sync_file_range` will need to wait on it.
There is a second change in this PR to use `fdatasync` when `sync_file_range` is unavailable (determined statically) or has some known problem with the underlying filesystem (determined dynamically).
The above two changes only apply when the user enables a new option, `strict_bytes_per_sync`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5183
Differential Revision: D14953553
Pulled By: siying
fbshipit-source-id: 445c3862e019fb7b470f9c7f314fc231b62706e9
2019-04-22 20:48:45 +02:00
|
|
|
env_options->strict_bytes_per_sync = options.strict_bytes_per_sync;
|
2019-10-14 21:23:39 +02:00
|
|
|
options.env->SanitizeEnvOptions(env_options);
|
2013-06-08 00:35:17 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2015-05-19 02:03:59 +02:00
|
|
|
EnvOptions Env::OptimizeForLogWrite(const EnvOptions& env_options,
|
|
|
|
const DBOptions& db_options) const {
|
|
|
|
EnvOptions optimized_env_options(env_options);
|
|
|
|
optimized_env_options.bytes_per_sync = db_options.wal_bytes_per_sync;
|
2017-10-31 21:49:25 +01:00
|
|
|
optimized_env_options.writable_file_max_buffer_size =
|
|
|
|
db_options.writable_file_max_buffer_size;
|
2015-05-19 02:03:59 +02:00
|
|
|
return optimized_env_options;
|
2014-03-18 05:52:14 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
EnvOptions Env::OptimizeForManifestWrite(const EnvOptions& env_options) const {
|
|
|
|
return env_options;
|
2014-02-28 22:19:47 +01:00
|
|
|
}
|
|
|
|
|
2017-05-23 03:40:41 +02:00
|
|
|
EnvOptions Env::OptimizeForLogRead(const EnvOptions& env_options) const {
|
|
|
|
EnvOptions optimized_env_options(env_options);
|
|
|
|
optimized_env_options.use_direct_reads = false;
|
|
|
|
return optimized_env_options;
|
|
|
|
}
|
|
|
|
|
|
|
|
EnvOptions Env::OptimizeForManifestRead(const EnvOptions& env_options) const {
|
|
|
|
EnvOptions optimized_env_options(env_options);
|
|
|
|
optimized_env_options.use_direct_reads = false;
|
|
|
|
return optimized_env_options;
|
|
|
|
}
|
|
|
|
|
2017-04-13 22:07:33 +02:00
|
|
|
EnvOptions Env::OptimizeForCompactionTableWrite(
|
|
|
|
const EnvOptions& env_options, const ImmutableDBOptions& db_options) const {
|
|
|
|
EnvOptions optimized_env_options(env_options);
|
|
|
|
optimized_env_options.use_direct_writes =
|
|
|
|
db_options.use_direct_io_for_flush_and_compaction;
|
|
|
|
return optimized_env_options;
|
|
|
|
}
|
|
|
|
|
|
|
|
EnvOptions Env::OptimizeForCompactionTableRead(
|
|
|
|
const EnvOptions& env_options, const ImmutableDBOptions& db_options) const {
|
|
|
|
EnvOptions optimized_env_options(env_options);
|
2018-05-10 04:26:43 +02:00
|
|
|
optimized_env_options.use_direct_reads = db_options.use_direct_reads;
|
2021-04-07 22:37:36 +02:00
|
|
|
return optimized_env_options;
|
|
|
|
}
|
|
|
|
EnvOptions Env::OptimizeForBlobFileRead(
|
|
|
|
const EnvOptions& env_options, const ImmutableDBOptions& db_options) const {
|
|
|
|
EnvOptions optimized_env_options(env_options);
|
|
|
|
optimized_env_options.use_direct_reads = db_options.use_direct_reads;
|
2017-04-13 22:07:33 +02:00
|
|
|
return optimized_env_options;
|
|
|
|
}
|
|
|
|
|
2014-02-05 22:12:23 +01:00
|
|
|
EnvOptions::EnvOptions(const DBOptions& options) {
|
2013-06-08 00:35:17 +02:00
|
|
|
AssignEnvOptions(this, options);
|
|
|
|
}
|
|
|
|
|
|
|
|
EnvOptions::EnvOptions() {
|
2014-02-05 22:12:23 +01:00
|
|
|
DBOptions options;
|
2013-06-08 00:35:17 +02:00
|
|
|
AssignEnvOptions(this, options);
|
|
|
|
}
|
|
|
|
|
2019-07-09 23:48:07 +02:00
|
|
|
Status NewEnvLogger(const std::string& fname, Env* env,
|
|
|
|
std::shared_ptr<Logger>* result) {
|
2021-01-29 07:08:46 +01:00
|
|
|
FileOptions options;
|
2019-07-09 23:48:07 +02:00
|
|
|
// TODO: Tune the buffer size.
|
|
|
|
options.writable_file_max_buffer_size = 1024 * 1024;
|
2021-01-29 07:08:46 +01:00
|
|
|
std::unique_ptr<FSWritableFile> writable_file;
|
|
|
|
const auto status = env->GetFileSystem()->NewWritableFile(
|
|
|
|
fname, options, &writable_file, nullptr);
|
2019-07-09 23:48:07 +02:00
|
|
|
if (!status.ok()) {
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
|
2021-01-29 07:08:46 +01:00
|
|
|
*result = std::make_shared<EnvLogger>(std::move(writable_file), fname,
|
|
|
|
options, env);
|
2019-07-09 23:48:07 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2013-06-08 00:35:17 +02:00
|
|
|
|
2020-03-24 05:50:42 +01:00
|
|
|
const std::shared_ptr<FileSystem>& Env::GetFileSystem() const {
|
|
|
|
return file_system_;
|
|
|
|
}
|
2021-01-26 07:07:26 +01:00
|
|
|
|
|
|
|
const std::shared_ptr<SystemClock>& Env::GetSystemClock() const {
|
|
|
|
return system_clock_;
|
|
|
|
}
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|