andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/facebook/rocksdb

Browse Source
This commit is contained in:
Ankit Gupta 2014-04-15 17:38:20 -07:00
commit 495fc80292
53 changed files with 636 additions and 398 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Makefile
View File

@ -15,7 +15,12 @@ endif
ifeq ($(MAKECMDGOALS),shared_lib)
PLATFORM_SHARED_LDFLAGS=-fPIC
OPT += -DNDEBUG
endif
ifeq ($(MAKECMDGOALS),static_lib)
OPT += -DNDEBUG
endif
#-----------------------------------------------
# detect what platform we're building on
@ -113,8 +118,7 @@ TOOLS = \
db_repl_stress \
blob_store_bench
PROGRAMS = db_bench signal_test table_reader_bench $(TESTS) $(TOOLS)
PROGRAMS = db_bench signal_test table_reader_bench $(TOOLS)
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db table_reader_bench
# The library name is configurable since we are maintaining libraries of both
@ -160,18 +164,18 @@ endif # PLATFORM_SHARED_EXT
release tags valgrind_check whitebox_crash_test format static_lib shared_lib all \
dbg
all: $(LIBRARY) $(PROGRAMS)
all: $(LIBRARY) $(PROGRAMS) $(TESTS)
static_lib: $(LIBRARY)
shared_lib: $(SHARED)
dbg: $(LIBRARY) $(PROGRAMS)
dbg: $(LIBRARY) $(PROGRAMS) $(TESTS)
# Will also generate shared libraries.
# creates static library and programs
release:
$(MAKE) clean
OPT="-DNDEBUG -O2" $(MAKE) all -j32
OPT="-DNDEBUG -O2" $(MAKE) static_lib $(PROGRAMS) -j32
coverage:
$(MAKE) clean
@ -184,7 +188,7 @@ check: $(PROGRAMS) $(TESTS) $(TOOLS)
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
python tools/ldb_test.py
ldb_tests: all $(PROGRAMS) $(TOOLS)
ldb_tests: all $(PROGRAMS) $(TESTS) $(TOOLS)
python tools/ldb_test.py
crash_test: blackbox_crash_test whitebox_crash_test
@ -220,7 +224,7 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
done
clean:
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) build_config.mk
-rm -f $(PROGRAMS) $(TESTS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) build_config.mk
-rm -rf ios-x86/* ios-arm/*
-find . -name "*.[od]" -exec rm {} \;
-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;

20
ROCKSDB_LITE.md Normal file
View File

@ -0,0 +1,20 @@
# RocksDBLite
RocksDBLite is a project focused on mobile use cases, which don't need a lot of fancy things we've built for server workloads and they are very sensitive to binary size. For that reason, we added a compile flag ROCKSDB_LITE that comments out a lot of the nonessential code and keeps the binary lean.
Some examples of the features disabled by ROCKSDB_LITE:
* compiled-in support for LDB tool
* No backupable DB
* No support for replication (which we provide in form of TrasactionalIterator)
* No advanced monitoring tools
* No special-purpose memtables that are highly optimized for specific use cases
When adding a new big feature to RocksDB, please add ROCKSDB_LITE compile guard if:
* Nobody from mobile really needs your feature,
* Your feature is adding a lot of weight to the binary.
Don't add ROCKSDB_LITE compile guard if:
* It would introduce a lot of code complexity. Compile guards make code harder to read. It's a trade-off.
* Your feature is not adding a lot of weight.
If unsure, ask. :)

2
build_tools/build_detect_platform
View File

@ -100,7 +100,7 @@ case "$TARGET_OS" in
;;
IOS)
PLATFORM=IOS
COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE"
COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE -DROCKSDB_LITE"
PLATFORM_SHARED_EXT=dylib
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
CROSS_COMPILE=true

4
db/c.cc
View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "rocksdb/c.h"
#include <stdlib.h>
@ -1467,3 +1469,5 @@ extern void rocksdb_livefiles_destroy(
}
} // end extern "C"
#endif // ROCKSDB_LITE

47
db/column_family_test.cc
View File

@ -29,15 +29,38 @@ std::string RandomString(Random* rnd, int len) {
}
} // anonymous namespace
// counts how many operations were performed
class EnvCounter : public EnvWrapper {
public:
explicit EnvCounter(Env* base)
: EnvWrapper(base), num_new_writable_file_(0) {}
int GetNumberOfNewWritableFileCalls() {
return num_new_writable_file_;
}
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
const EnvOptions& soptions) {
++num_new_writable_file_;
return EnvWrapper::NewWritableFile(f, r, soptions);
}
private:
int num_new_writable_file_;
};
class ColumnFamilyTest {
public:
ColumnFamilyTest() : rnd_(139) {
env_ = Env::Default();
env_ = new EnvCounter(Env::Default());
dbname_ = test::TmpDir() + "/column_family_test";
db_options_.create_if_missing = true;
db_options_.env = env_;
DestroyDB(dbname_, Options(db_options_, column_family_options_));
}
~ColumnFamilyTest() {
delete env_;
}
void Close() {
for (auto h : handles_) {
delete h;
@ -299,7 +322,7 @@ class ColumnFamilyTest {
DBOptions db_options_;
std::string dbname_;
DB* db_ = nullptr;
Env* env_;
EnvCounter* env_;
Random rnd_;
};
@ -895,6 +918,26 @@ TEST(ColumnFamilyTest, ReadOnlyDBTest) {
ASSERT_TRUE(!s.ok());
}
TEST(ColumnFamilyTest, DontRollEmptyLogs) {
Open();
CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
for (int i = 0; i < handles_.size(); ++i) {
PutRandomData(i, 10, 100);
}
int num_writable_file_start = env_->GetNumberOfNewWritableFileCalls();
// this will trigger the flushes
ASSERT_OK(db_->Write(WriteOptions(), nullptr));
for (int i = 0; i < 4; ++i) {
dbfull()->TEST_WaitForFlushMemTable(handles_[i]);
}
int total_new_writable_files =
env_->GetNumberOfNewWritableFileCalls() - num_writable_file_start;
ASSERT_EQ(total_new_writable_files, handles_.size() + 1);
Close();
}
} // namespace rocksdb
int main(int argc, char** argv) {

4
db/db_filesnapshot.cc
View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef ROCKSDB_LITE
#define __STDC_FORMAT_MACROS
#include <inttypes.h>
#include <algorithm>
@ -166,3 +168,5 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
}
}
#endif // ROCKSDB_LITE

592
db/db_impl.cc
View File

@ -349,6 +349,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
shutting_down_(nullptr),
bg_cv_(&mutex_),
logfile_number_(0),
log_empty_(true),
default_cf_handle_(nullptr),
tmp_batch_(),
bg_schedule_needed_(false),
@ -463,10 +464,6 @@ DBImpl::~DBImpl() {
LogFlush(options_.info_log);
}
uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
return versions_->ManifestFileNumber();
}
Status DBImpl::NewDB() {
VersionEdit new_db;
new_db.SetLogNumber(0);
@ -770,6 +767,7 @@ void DBImpl::DeleteObsoleteFiles() {
}
}
#ifndef ROCKSDB_LITE
// 1. Go through all archived files and
// a. if ttl is enabled, delete outdated files
// b. if archive size limit is enabled, delete empty files,
@ -894,11 +892,178 @@ void DBImpl::PurgeObsoleteWALFiles() {
}
}
namespace {
struct CompareLogByPointer {
bool operator()(const unique_ptr<LogFile>& a, const unique_ptr<LogFile>& b) {
LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
return *a_impl < *b_impl;
}
};
}
Status DBImpl::GetSortedWalsOfType(const std::string& path,
VectorLogPtr& log_files,
WalFileType log_type) {
std::vector<std::string> all_files;
const Status status = env_->GetChildren(path, &all_files);
if (!status.ok()) {
return status;
}
log_files.reserve(all_files.size());
for (const auto& f : all_files) {
uint64_t number;
FileType type;
if (ParseFileName(f, &number, &type) && type == kLogFile) {
WriteBatch batch;
Status s = ReadFirstRecord(log_type, number, &batch);
if (!s.ok()) {
if (CheckWalFileExistsAndEmpty(log_type, number)) {
continue;
}
return s;
}
uint64_t size_bytes;
s = env_->GetFileSize(LogFileName(path, number), &size_bytes);
if (!s.ok()) {
return s;
}
log_files.push_back(std::move(unique_ptr<LogFile>(
new LogFileImpl(number, log_type,
WriteBatchInternal::Sequence(&batch), size_bytes))));
}
}
CompareLogByPointer compare_log_files;
std::sort(log_files.begin(), log_files.end(), compare_log_files);
return status;
}
Status DBImpl::RetainProbableWalFiles(VectorLogPtr& all_logs,
const SequenceNumber target) {
int64_t start = 0; // signed to avoid overflow when target is < first file.
int64_t end = static_cast<int64_t>(all_logs.size()) - 1;
// Binary Search. avoid opening all files.
while (end >= start) {
int64_t mid = start + (end - start) / 2; // Avoid overflow.
SequenceNumber current_seq_num = all_logs.at(mid)->StartSequence();
if (current_seq_num == target) {
end = mid;
break;
} else if (current_seq_num < target) {
start = mid + 1;
} else {
end = mid - 1;
}
}
// end could be -ve.
size_t start_index = std::max(static_cast<int64_t>(0), end);
// The last wal file is always included
all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
return Status::OK();
}
bool DBImpl::CheckWalFileExistsAndEmpty(const WalFileType type,
const uint64_t number) {
const std::string fname = (type == kAliveLogFile)
? LogFileName(options_.wal_dir, number)
: ArchivedLogFileName(options_.wal_dir, number);
uint64_t file_size;
Status s = env_->GetFileSize(fname, &file_size);
return (s.ok() && (file_size == 0));
}
Status DBImpl::ReadFirstRecord(const WalFileType type, const uint64_t number,
WriteBatch* const result) {
if (type == kAliveLogFile) {
std::string fname = LogFileName(options_.wal_dir, number);
Status status = ReadFirstLine(fname, result);
if (status.ok() || env_->FileExists(fname)) {
// return OK or any error that is not caused non-existing file
return status;
}
// check if the file got moved to archive.
std::string archived_file = ArchivedLogFileName(options_.wal_dir, number);
Status s = ReadFirstLine(archived_file, result);
if (s.ok() || env_->FileExists(archived_file)) {
return s;
}
return Status::NotFound("Log File has been deleted: " + archived_file);
} else if (type == kArchivedLogFile) {
std::string fname = ArchivedLogFileName(options_.wal_dir, number);
Status status = ReadFirstLine(fname, result);
return status;
}
return Status::NotSupported("File Type Not Known: " + std::to_string(type));
}
Status DBImpl::ReadFirstLine(const std::string& fname,
WriteBatch* const batch) {
struct LogReporter : public log::Reader::Reporter {
Env* env;
Logger* info_log;
const char* fname;
Status* status;
bool ignore_error; // true if options_.paranoid_checks==false
virtual void Corruption(size_t bytes, const Status& s) {
Log(info_log, "%s%s: dropping %d bytes; %s",
(this->ignore_error ? "(ignoring error) " : ""), fname,
static_cast<int>(bytes), s.ToString().c_str());
if (this->status->ok()) {
// only keep the first error
*this->status = s;
}
}
};
unique_ptr<SequentialFile> file;
Status status = env_->NewSequentialFile(fname, &file, storage_options_);
if (!status.ok()) {
return status;
}
LogReporter reporter;
reporter.env = env_;
reporter.info_log = options_.info_log.get();
reporter.fname = fname.c_str();
reporter.status = &status;
reporter.ignore_error = !options_.paranoid_checks;
log::Reader reader(std::move(file), &reporter, true /*checksum*/,
0 /*initial_offset*/);
std::string scratch;
Slice record;
if (reader.ReadRecord(&record, &scratch) &&
(status.ok() || !options_.paranoid_checks)) {
if (record.size() < 12) {
reporter.Corruption(record.size(),
Status::Corruption("log record too small"));
// TODO read record's till the first no corrupt entry?
} else {
WriteBatchInternal::SetContents(batch, record);
return Status::OK();
}
}
// ReadRecord returns false on EOF, which is deemed as OK() by Reader
if (status.ok()) {
status = Status::Corruption("eof reached");
}
return status;
}
#endif // ROCKSDB_LITE
Status DBImpl::Recover(
const std::vector<ColumnFamilyDescriptor>& column_families, bool read_only,
bool error_if_log_file_exist) {
mutex_.AssertHeld();
bool is_new_db = false;
assert(db_lock_ == nullptr);
if (!read_only) {
// We call CreateDirIfMissing() as the directory may already exist (if we
@ -927,6 +1092,7 @@ Status DBImpl::Recover(
if (options_.create_if_missing) {
// TODO: add merge_operator name check
s = NewDB();
is_new_db = true;
if (!s.ok()) {
return s;
}
@ -977,10 +1143,15 @@ Status DBImpl::Recover(
for (size_t i = 0; i < filenames.size(); i++) {
uint64_t number;
FileType type;
if (ParseFileName(filenames[i], &number, &type)
&& type == kLogFile
&& ((number >= min_log) || (number == prev_log))) {
logs.push_back(number);
if (ParseFileName(filenames[i], &number, &type) && type == kLogFile) {
if (is_new_db) {
return Status::Corruption(
"While creating a new Db, wal_dir contains "
"existing log file: ",
filenames[i]);
} else if ((number >= min_log) || (number == prev_log)) {
logs.push_back(number);
}
}
}
@ -1532,198 +1703,6 @@ SequenceNumber DBImpl::GetLatestSequenceNumber() const {
return versions_->LastSequence();
}
Status DBImpl::GetUpdatesSince(
SequenceNumber seq, unique_ptr<TransactionLogIterator>* iter,
const TransactionLogIterator::ReadOptions& read_options) {
RecordTick(options_.statistics.get(), GET_UPDATES_SINCE_CALLS);
if (seq > versions_->LastSequence()) {
return Status::NotFound(
"Requested sequence not yet written in the db");
}
// Get all sorted Wal Files.
// Do binary search and open files and find the seq number.
std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
Status s = GetSortedWalFiles(*wal_files);
if (!s.ok()) {
return s;
}
s = RetainProbableWalFiles(*wal_files, seq);
if (!s.ok()) {
return s;
}
iter->reset(new TransactionLogIteratorImpl(options_.wal_dir, &options_,
read_options, storage_options_,
seq, std::move(wal_files), this));
return (*iter)->status();
}
Status DBImpl::RetainProbableWalFiles(VectorLogPtr& all_logs,
const SequenceNumber target) {
long start = 0; // signed to avoid overflow when target is < first file.
long end = static_cast<long>(all_logs.size()) - 1;
// Binary Search. avoid opening all files.
while (end >= start) {
long mid = start + (end - start) / 2; // Avoid overflow.
SequenceNumber current_seq_num = all_logs.at(mid)->StartSequence();
if (current_seq_num == target) {
end = mid;
break;
} else if (current_seq_num < target) {
start = mid + 1;
} else {
end = mid - 1;
}
}
size_t start_index = std::max(0l, end); // end could be -ve.
// The last wal file is always included
all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
return Status::OK();
}
bool DBImpl::CheckWalFileExistsAndEmpty(const WalFileType type,
const uint64_t number) {
const std::string fname = (type == kAliveLogFile) ?
LogFileName(options_.wal_dir, number) :
ArchivedLogFileName(options_.wal_dir, number);
uint64_t file_size;
Status s = env_->GetFileSize(fname, &file_size);
return (s.ok() && (file_size == 0));
}
Status DBImpl::ReadFirstRecord(const WalFileType type, const uint64_t number,
WriteBatch* const result) {
if (type == kAliveLogFile) {
std::string fname = LogFileName(options_.wal_dir, number);
Status status = ReadFirstLine(fname, result);
if (status.ok() || env_->FileExists(fname)) {
// return OK or any error that is not caused non-existing file
return status;
}
// check if the file got moved to archive.
std::string archived_file =
ArchivedLogFileName(options_.wal_dir, number);
Status s = ReadFirstLine(archived_file, result);
if (s.ok() || env_->FileExists(archived_file)) {
return s;
}
return Status::NotFound("Log File has been deleted: " + archived_file);
} else if (type == kArchivedLogFile) {
std::string fname = ArchivedLogFileName(options_.wal_dir, number);
Status status = ReadFirstLine(fname, result);
return status;
}
return Status::NotSupported("File Type Not Known: " + std::to_string(type));
}
Status DBImpl::ReadFirstLine(const std::string& fname,
WriteBatch* const batch) {
struct LogReporter : public log::Reader::Reporter {
Env* env;
Logger* info_log;
const char* fname;
Status* status;
bool ignore_error; // true if options_.paranoid_checks==false
virtual void Corruption(size_t bytes, const Status& s) {
Log(info_log, "%s%s: dropping %d bytes; %s",
(this->ignore_error ? "(ignoring error) " : ""),
fname, static_cast<int>(bytes), s.ToString().c_str());
if (this->status->ok()) {
// only keep the first error
*this->status = s;
}
}
};
unique_ptr<SequentialFile> file;
Status status = env_->NewSequentialFile(fname, &file, storage_options_);
if (!status.ok()) {
return status;
}
LogReporter reporter;
reporter.env = env_;
reporter.info_log = options_.info_log.get();
reporter.fname = fname.c_str();
reporter.status = &status;
reporter.ignore_error = !options_.paranoid_checks;
log::Reader reader(std::move(file), &reporter, true/*checksum*/,
0/*initial_offset*/);
std::string scratch;
Slice record;
if (reader.ReadRecord(&record, &scratch) &&
(status.ok() || !options_.paranoid_checks)) {
if (record.size() < 12) {
reporter.Corruption(
record.size(), Status::Corruption("log record too small"));
// TODO read record's till the first no corrupt entry?
} else {
WriteBatchInternal::SetContents(batch, record);
return Status::OK();
}
}
// ReadRecord returns false on EOF, which is deemed as OK() by Reader
if (status.ok()) {
status = Status::Corruption("eof reached");
}
return status;
}
struct CompareLogByPointer {
bool operator() (const unique_ptr<LogFile>& a,
const unique_ptr<LogFile>& b) {
LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
return *a_impl < *b_impl;
}
};
Status DBImpl::GetSortedWalsOfType(const std::string& path,
VectorLogPtr& log_files, WalFileType log_type) {
std::vector<std::string> all_files;
const Status status = env_->GetChildren(path, &all_files);
if (!status.ok()) {
return status;
}
log_files.reserve(all_files.size());
for (const auto& f : all_files) {
uint64_t number;
FileType type;
if (ParseFileName(f, &number, &type) && type == kLogFile){
WriteBatch batch;
Status s = ReadFirstRecord(log_type, number, &batch);
if (!s.ok()) {
if (CheckWalFileExistsAndEmpty(log_type, number)) {
continue;
}
return s;
}
uint64_t size_bytes;
s = env_->GetFileSize(LogFileName(path, number), &size_bytes);
if (!s.ok()) {
return s;
}
log_files.push_back(std::move(unique_ptr<LogFile>(new LogFileImpl(
number, log_type, WriteBatchInternal::Sequence(&batch), size_bytes))));
}
}
CompareLogByPointer compare_log_files;
std::sort(log_files.begin(), log_files.end(), compare_log_files);
return status;
}
Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
int output_level, const Slice* begin,
const Slice* end) {
@ -1797,23 +1776,6 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
return manual.status;
}
Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
const Slice* end,
ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
int output_level =
(cfd->options()->compaction_style == kCompactionStyleUniversal)
? level
: level + 1;
return RunManualCompaction(cfd, level, output_level, begin, end);
}
Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
const FlushOptions& options) {
// nullptr batch means just wait for earlier writes to be done
@ -1838,38 +1800,6 @@ Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd) {
return s;
}
Status DBImpl::TEST_FlushMemTable(bool wait) {
FlushOptions fo;
fo.wait = wait;
return FlushMemTable(default_cf_handle_->cfd(), fo);
}
Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
return WaitForFlushMemTable(cfd);
}
Status DBImpl::TEST_WaitForCompact() {
// Wait until the compaction completes
// TODO: a bug here. This function actually does not necessarily
// wait for compact. It actually waits for scheduled compaction
// OR flush to finish.
MutexLock l(&mutex_);
while ((bg_compaction_scheduled_ || bg_flush_scheduled_) &&
bg_error_.ok()) {
bg_cv_.Wait();
}
return bg_error_;
}
void DBImpl::MaybeScheduleFlushOrCompaction() {
mutex_.AssertHeld();
bg_schedule_needed_ = false;
@ -2025,16 +1955,6 @@ void DBImpl::BackgroundCallFlush() {
}
}
void DBImpl::TEST_PurgeObsoleteteWAL() {
PurgeObsoleteWALFiles();
}
uint64_t DBImpl::TEST_GetLevel0TotalSize() {
MutexLock l(&mutex_);
return default_cf_handle_->cfd()->current()->NumLevelBytes(0);
}
void DBImpl::BackgroundCallCompaction() {
bool madeProgress = false;
DeletionState deletion_state(true);
@ -3221,36 +3141,6 @@ ColumnFamilyHandle* DBImpl::DefaultColumnFamily() const {
return default_cf_handle_;
}
Iterator* DBImpl::TEST_NewInternalIterator(ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
mutex_.Lock();
SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
mutex_.Unlock();
ReadOptions roptions;
roptions.prefix_seek = true;
return NewInternalIterator(roptions, cfd, super_version);
}
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
MutexLock l(&mutex_);
return cfd->current()->MaxNextLevelOverlappingBytes();
}
Status DBImpl::Get(const ReadOptions& options,
ColumnFamilyHandle* column_family, const Slice& key,
std::string* value) {
@ -3589,7 +3479,12 @@ Iterator* DBImpl::NewIterator(const ReadOptions& options,
Iterator* iter;
if (options.tailing) {
#ifdef ROCKSDB_LITE
// not supported in lite version
return nullptr;
#else
iter = new TailingIterator(env_, this, options, cfd);
#endif
} else {
SequenceNumber latest_snapshot = versions_->LastSequence();
SuperVersion* sv = nullptr;
@ -3641,10 +3536,15 @@ Status DBImpl::NewIterators(
}
if (options.tailing) {
#ifdef ROCKSDB_LITE
return Status::InvalidArgument(
"Tailing interator not supported in RocksDB lite");
#else
for (auto cfh : column_families) {
auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
iterators->push_back(new TailingIterator(env_, this, options, cfd));
}
#endif
} else {
for (size_t i = 0; i < column_families.size(); ++i) {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_families[i]);
@ -3782,6 +3682,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
PERF_TIMER_START(write_wal_time);
Slice log_entry = WriteBatchInternal::Contents(updates);
status = log_->AddRecord(log_entry);
log_empty_ = false;
RecordTick(options_.statistics.get(), WAL_FILE_SYNCED, 1);
RecordTick(options_.statistics.get(), WAL_FILE_BYTES, log_entry.size());
if (status.ok() && options.sync) {
@ -4057,57 +3958,66 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) {
// Attempt to switch to a new memtable and trigger flush of old.
// Do this without holding the dbmutex lock.
assert(versions_->PrevLogNumber() == 0);
uint64_t new_log_number = versions_->NewFileNumber();
bool creating_new_log = !log_empty_;
uint64_t new_log_number =
creating_new_log ? versions_->NewFileNumber() : logfile_number_;
SuperVersion* new_superversion = nullptr;
mutex_.Unlock();
{
DelayLoggingAndReset();
s = env_->NewWritableFile(LogFileName(options_.wal_dir, new_log_number),
&lfile,
env_->OptimizeForLogWrite(storage_options_));
if (creating_new_log) {
s = env_->NewWritableFile(
LogFileName(options_.wal_dir, new_log_number), &lfile,
env_->OptimizeForLogWrite(storage_options_));
if (s.ok()) {
// Our final size should be less than write_buffer_size
// (compression, etc) but err on the side of caution.
lfile->SetPreallocationBlockSize(1.1 *
cfd->options()->write_buffer_size);
new_log = new log::Writer(std::move(lfile));
}
}
if (s.ok()) {
// Our final size should be less than write_buffer_size
// (compression, etc) but err on the side of caution.
lfile->SetPreallocationBlockSize(1.1 *
cfd->options()->write_buffer_size);
new_log = new log::Writer(std::move(lfile));
new_mem = new MemTable(cfd->internal_comparator(), *cfd->options());
new_superversion = new SuperVersion();
}
Log(options_.info_log,
"New memtable created with log file: #%lu\n",
(unsigned long)new_log_number);
}
mutex_.Lock();
if (!s.ok()) {
// how do we fail if we're not creating new log?
assert(creating_new_log);
// Avoid chewing through file number space in a tight loop.
versions_->ReuseFileNumber(new_log_number);
assert(!new_mem);
assert(!new_log);
break;
}
logfile_number_ = new_log_number;
assert(new_log != nullptr);
// TODO(icanadi) delete outside of mutex
delete log_.release();
log_.reset(new_log);
if (creating_new_log) {
logfile_number_ = new_log_number;
assert(new_log != nullptr);
// TODO(icanadi) delete outside of mutex
delete log_.release();
log_.reset(new_log);
log_empty_ = true;
alive_log_files_.push_back(logfile_number_);
for (auto cfd : *versions_->GetColumnFamilySet()) {
// all this is just optimization to delete logs that
// are no longer needed -- if CF is empty, that means it
// doesn't need that particular log to stay alive, so we just
// advance the log number. no need to persist this in the manifest
if (cfd->mem()->GetFirstSequenceNumber() == 0 &&
cfd->imm()->size() == 0) {
cfd->SetLogNumber(logfile_number_);
}
}
}
cfd->mem()->SetNextLogNumber(logfile_number_);
cfd->imm()->Add(cfd->mem());
if (force) {
cfd->imm()->FlushRequested();
}
new_mem->Ref();
alive_log_files_.push_back(logfile_number_);
for (auto cfd : *versions_->GetColumnFamilySet()) {
// all this is just optimization to delete logs that
// are no longer needed -- if CF is empty, that means it
// doesn't need that particular log to stay alive, so we just
// advance the log number. no need to persist this in the manifest
if (cfd->mem()->GetFirstSequenceNumber() == 0 &&
cfd->imm()->size() == 0) {
cfd->SetLogNumber(logfile_number_);
}
}
cfd->SetMemtable(new_mem);
Log(options_.info_log,
"[CF %" PRIu32 "] New memtable created with log file: #%lu\n",
@ -4121,6 +4031,7 @@ Status DBImpl::MakeRoomForWrite(ColumnFamilyData* cfd, bool force) {
return s;
}
#ifndef ROCKSDB_LITE
Status DBImpl::GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
TablePropertiesCollection* props) {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
@ -4141,6 +4052,7 @@ Status DBImpl::GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
return s;
}
#endif // ROCKSDB_LITE
const std::string& DBImpl::GetName() const {
return dbname_;
@ -4200,6 +4112,34 @@ inline void DBImpl::DelayLoggingAndReset() {
}
}
#ifndef ROCKSDB_LITE
Status DBImpl::GetUpdatesSince(
SequenceNumber seq, unique_ptr<TransactionLogIterator>* iter,
const TransactionLogIterator::ReadOptions& read_options) {
RecordTick(options_.statistics.get(), GET_UPDATES_SINCE_CALLS);
if (seq > versions_->LastSequence()) {
return Status::NotFound("Requested sequence not yet written in the db");
}
// Get all sorted Wal Files.
// Do binary search and open files and find the seq number.
std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
Status s = GetSortedWalFiles(*wal_files);
if (!s.ok()) {
return s;
}
s = RetainProbableWalFiles(*wal_files, seq);
if (!s.ok()) {
return s;
}
iter->reset(new TransactionLogIteratorImpl(options_.wal_dir, &options_,
read_options, storage_options_,
seq, std::move(wal_files), this));
return (*iter)->status();
}
Status DBImpl::DeleteFile(std::string name) {
uint64_t number;
FileType type;
@ -4283,6 +4223,7 @@ void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
MutexLock l(&mutex_);
versions_->GetLiveFilesMetaData(metadata);
}
#endif // ROCKSDB_LITE
Status DBImpl::CheckConsistency() {
mutex_.AssertHeld();
@ -4311,23 +4252,6 @@ Status DBImpl::CheckConsistency() {
}
}
void DBImpl::TEST_GetFilesMetaData(
ColumnFamilyHandle* column_family,
std::vector<std::vector<FileMetaData>>* metadata) {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
auto cfd = cfh->cfd();
MutexLock l(&mutex_);
metadata->resize(NumberLevels());
for (int level = 0; level < NumberLevels(); level++) {
const std::vector<FileMetaData*>& files = cfd->current()->files_[level];
(*metadata)[level].clear();
for (const auto& f : files) {
(*metadata)[level].push_back(*f);
}
}
}
Status DBImpl::GetDbIdentity(std::string& identity) {
std::string idfilename = IdentityFileName(dbname_);
unique_ptr<SequentialFile> idfile;

26
db/db_impl.h
View File

@ -115,6 +115,10 @@ class DBImpl : public DB {
using DB::Flush;
virtual Status Flush(const FlushOptions& options,
ColumnFamilyHandle* column_family);
virtual SequenceNumber GetLatestSequenceNumber() const;
#ifndef ROCKSDB_LITE
virtual Status DisableFileDeletions();
virtual Status EnableFileDeletions(bool force);
// All the returned filenames start with "/"
@ -122,7 +126,7 @@ class DBImpl : public DB {
uint64_t* manifest_file_size,
bool flush_memtable = true);
virtual Status GetSortedWalFiles(VectorLogPtr& files);
virtual SequenceNumber GetLatestSequenceNumber() const;
virtual Status GetUpdatesSince(
SequenceNumber seq_number, unique_ptr<TransactionLogIterator>* iter,
const TransactionLogIterator::ReadOptions&
@ -130,6 +134,7 @@ class DBImpl : public DB {
virtual Status DeleteFile(std::string name);
virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
#endif // ROCKSDB_LITE
// checks if all live files exist on file system and that their file sizes
// match to our in-memory records
@ -141,7 +146,9 @@ class DBImpl : public DB {
int output_level, const Slice* begin,
const Slice* end);
#ifndef ROCKSDB_LITE
// Extra methods (for testing) that are not in the public DB interface
// Implemented in db_impl_debug.cc
// Compact any files in the named level that overlap [*begin, *end]
Status TEST_CompactRange(int level, const Slice* begin, const Slice* end,
@ -184,6 +191,8 @@ class DBImpl : public DB {
void TEST_GetFilesMetaData(ColumnFamilyHandle* column_family,
std::vector<std::vector<FileMetaData>>* metadata);
#endif // NDEBUG
// needed for CleanupIteratorState
struct DeletionState {
inline bool HaveSomethingToDelete() const {
@ -270,7 +279,9 @@ class DBImpl : public DB {
private:
friend class DB;
friend class InternalStats;
#ifndef ROCKSDB_LITE
friend class TailingIterator;
#endif
friend struct SuperVersion;
struct CompactionState;
struct Writer;
@ -326,8 +337,11 @@ class DBImpl : public DB {
Status WaitForFlushMemTable(ColumnFamilyData* cfd);
void MaybeScheduleLogDBDeployStats();
#ifndef ROCKSDB_LITE
static void BGLogDBDeployStats(void* db);
void LogDBDeployStats();
#endif // ROCKSDB_LITE
void MaybeScheduleFlushOrCompaction();
static void BGWorkCompaction(void* db);
@ -375,6 +389,12 @@ class DBImpl : public DB {
void AllocateCompactionOutputFileNumbers(CompactionState* compact);
void ReleaseCompactionUnusedFileNumbers(CompactionState* compact);
#ifdef ROCKSDB_LITE
void PurgeObsoleteWALFiles() {
// this function is used for archiving WAL files. we don't need this in
// ROCKSDB_LITE
}
#else
void PurgeObsoleteWALFiles();
Status GetSortedWalsOfType(const std::string& path,
@ -394,6 +414,7 @@ class DBImpl : public DB {
WriteBatch* const result);
Status ReadFirstLine(const std::string& fname, WriteBatch* const batch);
#endif // ROCKSDB_LITE
void PrintStatistics();
@ -421,6 +442,7 @@ class DBImpl : public DB {
port::CondVar bg_cv_; // Signalled when background work finishes
uint64_t logfile_number_;
unique_ptr<log::Writer> log_;
bool log_empty_;
ColumnFamilyHandleImpl* default_cf_handle_;
unique_ptr<ColumnFamilyMemTablesImpl> column_family_memtables_;
std::deque<uint64_t> alive_log_files_;
@ -539,10 +561,12 @@ class DBImpl : public DB {
void InstallSuperVersion(ColumnFamilyData* cfd,
DeletionState& deletion_state);
#ifndef ROCKSDB_LITE
using DB::GetPropertiesOfAllTables;
virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
TablePropertiesCollection* props)
override;
#endif // ROCKSDB_LITE
// Function that Get and KeyMayExist call with no_io true or false
// Note: 'value_found' from KeyMayExist propagates here

122
db/db_impl_debug.cc Normal file
View File

@ -0,0 +1,122 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "db/db_impl.h"
namespace rocksdb {
void DBImpl::TEST_PurgeObsoleteteWAL() { PurgeObsoleteWALFiles(); }
uint64_t DBImpl::TEST_GetLevel0TotalSize() {
MutexLock l(&mutex_);
return default_cf_handle_->cfd()->current()->NumLevelBytes(0);
}
Iterator* DBImpl::TEST_NewInternalIterator(ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
mutex_.Lock();
SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
mutex_.Unlock();
ReadOptions roptions;
roptions.prefix_seek = true;
return NewInternalIterator(roptions, cfd, super_version);
}
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
MutexLock l(&mutex_);
return cfd->current()->MaxNextLevelOverlappingBytes();
}
void DBImpl::TEST_GetFilesMetaData(
ColumnFamilyHandle* column_family,
std::vector<std::vector<FileMetaData>>* metadata) {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
auto cfd = cfh->cfd();
MutexLock l(&mutex_);
metadata->resize(NumberLevels());
for (int level = 0; level < NumberLevels(); level++) {
const std::vector<FileMetaData*>& files = cfd->current()->files_[level];
(*metadata)[level].clear();
for (const auto& f : files) {
(*metadata)[level].push_back(*f);
}
}
}
uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
return versions_->ManifestFileNumber();
}
Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
const Slice* end,
ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
int output_level =
(cfd->options()->compaction_style == kCompactionStyleUniversal)
? level
: level + 1;
return RunManualCompaction(cfd, level, output_level, begin, end);
}
Status DBImpl::TEST_FlushMemTable(bool wait) {
FlushOptions fo;
fo.wait = wait;
return FlushMemTable(default_cf_handle_->cfd(), fo);
}
Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
ColumnFamilyData* cfd;
if (column_family == nullptr) {
cfd = default_cf_handle_->cfd();
} else {
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
cfd = cfh->cfd();
}
return WaitForFlushMemTable(cfd);
}
Status DBImpl::TEST_WaitForCompact() {
// Wait until the compaction completes
// TODO: a bug here. This function actually does not necessarily
// wait for compact. It actually waits for scheduled compaction
// OR flush to finish.
MutexLock l(&mutex_);
while ((bg_compaction_scheduled_ || bg_flush_scheduled_) && bg_error_.ok()) {
bg_cv_.Wait();
}
return bg_error_;
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

5
db/db_stats_logger.cc
View File

@ -20,7 +20,8 @@
namespace rocksdb {
void DBImpl::MaybeScheduleLogDBDeployStats() {
// we did say maybe
#ifndef ROCKSDB_LITE
// There is a lock in the actual logger.
if (!logger_ || options_.db_stats_log_interval < 0
|| host_name_.empty()) {
@ -89,6 +90,6 @@ void DBImpl::LogDBDeployStats() {
bg_logstats_scheduled_ = false;
bg_cv_.SignalAll();
mutex_.Unlock();
#endif
}
}

26
db/db_test.cc
View File

@ -987,7 +987,6 @@ void VerifyTableProperties(DB* db, uint64_t expected_entries_size) {
TablePropertiesCollection props;
ASSERT_OK(db->GetPropertiesOfAllTables(&props));
assert(props.size() == 4);
ASSERT_EQ(4U, props.size());
std::unordered_set<uint64_t> unique_entries;
@ -2096,7 +2095,24 @@ TEST(DBTest, IgnoreRecoveredLog) {
ASSERT_EQ(one, Get("bar"));
Close();
Destroy(&options);
Reopen(&options);
Close();
// copy the logs from backup back to wal dir
env_->CreateDirIfMissing(options.wal_dir);
for (auto& log : logs) {
if (log != ".." && log != ".") {
CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
}
}
// assert that we successfully recovered only from logs, even though we
// destroyed the DB
Reopen(&options);
ASSERT_EQ(two, Get("foo"));
ASSERT_EQ(one, Get("bar"));
// Recovery will fail if DB directory doesn't exist.
Destroy(&options);
// copy the logs from backup back to wal dir
env_->CreateDirIfMissing(options.wal_dir);
for (auto& log : logs) {
@ -2106,12 +2122,8 @@ TEST(DBTest, IgnoreRecoveredLog) {
env_->DeleteFile(backup_logs + "/" + log);
}
}
// assert that we successfully recovered only from logs, even though we
// destroyed the DB
Reopen(&options);
ASSERT_EQ(two, Get("foo"));
ASSERT_EQ(one, Get("bar"));
Close();
Status s = TryReopen(&options);
ASSERT_TRUE(!s.ok());
} while (ChangeOptions());
}

4
db/repair.cc
View File

@ -29,6 +29,8 @@
// Store per-table metadata (smallest, largest, largest-seq#, ...)
// in the table's meta section to speed up ScanTable.
#ifndef ROCKSDB_LITE
#include "db/builder.h"
#include "db/db_impl.h"
#include "db/dbformat.h"
@ -396,3 +398,5 @@ Status RepairDB(const std::string& dbname, const Options& options) {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
db/tailing_iter.cc
View File

@ -3,6 +3,7 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef ROCKSDB_LITE
#include "db/tailing_iter.h"
#include <string>
@ -217,3 +218,4 @@ void TailingIterator::SeekImmutable(const Slice& target) {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
db/tailing_iter.h
View File

@ -2,6 +2,7 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
@ -92,3 +93,4 @@ class TailingIterator : public Iterator {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

4
db/transaction_log_impl.cc
View File

@ -2,7 +2,8 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "db/transaction_log_impl.h"
#include "db/write_batch_internal.h"
@ -257,3 +258,4 @@ Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
return Status::OK();
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

4
db/transaction_log_impl.h
View File

@ -2,7 +2,8 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#pragma once
#include <vector>
@ -116,3 +117,4 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
Status OpenLogReader(const LogFile* file);
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

4
db/version_set.cc
View File

@ -1742,7 +1742,7 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
}
}
if (max_log_number_in_batch != 0) {
assert(column_family_data->GetLogNumber() < max_log_number_in_batch);
assert(column_family_data->GetLogNumber() <= max_log_number_in_batch);
column_family_data->SetLogNumber(max_log_number_in_batch);
}
AppendVersion(column_family_data, v);
@ -2170,6 +2170,7 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
return s;
}
#ifndef ROCKSDB_LITE
Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
const Options* options,
const EnvOptions& storage_options,
@ -2430,6 +2431,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
return s;
}
#endif // ROCKSDB_LITE
void VersionSet::MarkFileNumberUsed(uint64_t number) {
if (next_file_number_ <= number) {

11
db/version_set.h
View File

@ -319,6 +319,7 @@ class VersionSet {
static Status ListColumnFamilies(std::vector<std::string>* column_families,
const std::string& dbname, Env* env);
#ifndef ROCKSDB_LITE
// Try to reduce the number of levels. This call is valid when
// only one level from the new max level to the old
// max level containing files.
@ -333,6 +334,12 @@ class VersionSet {
const EnvOptions& storage_options,
int new_levels);
// printf contents (for debugging)
Status DumpManifest(Options& options, std::string& manifestFileName,
bool verbose, bool hex = false);
#endif // ROCKSDB_LITE
// Return the current manifest file number
uint64_t ManifestFileNumber() const { return manifest_file_number_; }
@ -393,10 +400,6 @@ class VersionSet {
// "key" as of version "v".
uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
// printf contents (for debugging)
Status DumpManifest(Options& options, std::string& manifestFileName,
bool verbose, bool hex = false);
// Return the size of the current manifest file
uint64_t ManifestFileSize() const { return manifest_file_size_; }

14
include/rocksdb/db.h
View File

@ -381,6 +381,11 @@ class DB {
return Flush(options, DefaultColumnFamily());
}
// The sequence number of the most recent transaction.
virtual SequenceNumber GetLatestSequenceNumber() const = 0;
#ifndef ROCKSDB_LITE
// Prevent file deletions. Compactions will continue to occur,
// but no obsolete files will be deleted. Calling this multiple
// times have the same effect as calling it once.
@ -422,9 +427,6 @@ class DB {
// Retrieve the sorted list of all wal files with earliest file first
virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
// The sequence number of the most recent transaction.
virtual SequenceNumber GetLatestSequenceNumber() const = 0;
// Sets iter to an iterator that is positioned at a write-batch containing
// seq_number. If the sequence number is non existent, it returns an iterator
// at the first available seq_no after the requested seq_no
@ -447,6 +449,8 @@ class DB {
// and end key
virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {}
#endif // ROCKSDB_LITE
// Sets the globally unique ID created at database creation time by invoking
// Env::GenerateUniqueId(), in identity. Returns Status::OK if identity could
// be set properly
@ -455,11 +459,13 @@ class DB {
// Returns default column family handle
virtual ColumnFamilyHandle* DefaultColumnFamily() const = 0;
#ifndef ROCKSDB_LITE
virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
TablePropertiesCollection* props) = 0;
Status GetPropertiesOfAllTables(TablePropertiesCollection* props) {
return GetPropertiesOfAllTables(DefaultColumnFamily(), props);
}
#endif // ROCKSDB_LITE
private:
// No copying allowed
@ -471,11 +477,13 @@ class DB {
// Be very careful using this method.
Status DestroyDB(const std::string& name, const Options& options);
#ifndef ROCKSDB_LITE
// If a DB cannot be opened, you may attempt to call this method to
// resurrect as much of the contents of the database as possible.
// Some data may be lost, so be careful when calling this function
// on a database that contains important information.
Status RepairDB(const std::string& dbname, const Options& options);
#endif
} // namespace rocksdb

6
include/rocksdb/ldb_tool.h
View File

@ -2,8 +2,8 @@
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#ifndef STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
#define STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
#ifndef ROCKSDB_LITE
#pragma once
#include "rocksdb/options.h"
namespace rocksdb {
@ -15,4 +15,4 @@ class LDBTool {
} // namespace rocksdb
#endif // STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
#endif // ROCKSDB_LITE

23
include/rocksdb/memtablerep.h
View File

@ -177,6 +177,16 @@ class MemTableRepFactory {
virtual const char* Name() const = 0;
};
// This uses a skip list to store keys. It is the default.
class SkipListFactory : public MemTableRepFactory {
public:
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Arena*,
const SliceTransform*) override;
virtual const char* Name() const override { return "SkipListFactory"; }
};
#ifndef ROCKSDB_LITE
// This creates MemTableReps that are backed by an std::vector. On iteration,
// the vector is sorted. This is useful for workloads where iteration is very
// rare and writes are generally not issued after reads begin.
@ -198,17 +208,6 @@ class VectorRepFactory : public MemTableRepFactory {
}
};
// This uses a skip list to store keys. It is the default.
class SkipListFactory : public MemTableRepFactory {
public:
virtual MemTableRep* CreateMemTableRep(
const MemTableRep::KeyComparator&, Arena*,
const SliceTransform*) override;
virtual const char* Name() const override {
return "SkipListFactory";
}
};
// This class contains a fixed array of buckets, each
// pointing to a skiplist (null if the bucket is empty).
// bucket_count: number of fixed array buckets
@ -227,4 +226,6 @@ extern MemTableRepFactory* NewHashSkipListRepFactory(
extern MemTableRepFactory* NewHashLinkListRepFactory(
size_t bucket_count = 50000);
#endif // ROCKSDB_LITE
} // namespace rocksdb

1
include/rocksdb/options.h
View File

@ -850,6 +850,7 @@ struct ReadOptions {
// added data) and is optimized for sequential reads. It will return records
// that were inserted into the database after the creation of the iterator.
// Default: false
// Not supported in ROCKSDB_LITE mode!
bool tailing;
ReadOptions()

3
include/rocksdb/table.h
View File

@ -81,6 +81,7 @@ struct BlockBasedTablePropertyNames {
extern TableFactory* NewBlockBasedTableFactory(
const BlockBasedTableOptions& table_options = BlockBasedTableOptions());
#ifndef ROCKSDB_LITE
// -- Plain Table with prefix-only seek
// For this factory, you need to set Options.prefix_extrator properly to make it
// work. Look-up will starts with prefix hash lookup for key prefix. Inside the
@ -120,6 +121,8 @@ extern TableFactory* NewTotalOrderPlainTableFactory(
uint32_t user_key_len = kPlainTableVariableLength,
int bloom_bits_per_key = 0, size_t index_sparseness = 16);
#endif // ROCKSDB_LITE
// A base class for table factories.
class TableFactory {
public:

2
include/utilities/backupable_db.h
View File

@ -7,6 +7,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include "utilities/stackable_db.h"
#include "rocksdb/env.h"
@ -210,3 +211,4 @@ class RestoreBackupableDB {
};
} // rocksdb namespace
#endif // ROCKSDB_LITE

2
include/utilities/geo_db.h
View File

@ -4,6 +4,7 @@
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
#include <vector>
@ -101,3 +102,4 @@ class GeoDB : public StackableDB {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
include/utilities/utility_db.h
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include "stackable_db.h"
@ -48,3 +49,4 @@ class UtilityDB {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
table/plain_table_builder.cc
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "table/plain_table_builder.h"
#include <assert.h>
@ -205,3 +206,4 @@ uint64_t PlainTableBuilder::FileSize() const {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
table/plain_table_builder.h
View File

@ -5,6 +5,7 @@
// IndexedTable is a simple table format for UNIT TEST ONLY. It is not built
// as production quality.
#ifndef ROCKSDB_LITE
#pragma once
#include <stdint.h>
#include "rocksdb/options.h"
@ -80,3 +81,4 @@ private:
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
table/plain_table_factory.cc
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "table/plain_table_factory.h"
#include <memory>
@ -46,3 +47,4 @@ extern TableFactory* NewTotalOrderPlainTableFactory(uint32_t user_key_len,
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
table/plain_table_factory.h
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include <memory>
#include <stdint.h>
@ -83,3 +84,4 @@ class PlainTableFactory : public TableFactory {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
table/plain_table_reader.cc
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "table/plain_table_reader.h"
#include <string>
@ -745,3 +746,4 @@ Status PlainTableIterator::status() const {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
table/plain_table_reader.h
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include <unordered_map>
#include <memory>
@ -255,3 +256,4 @@ class PlainTableReader: public TableReader {
void operator=(const TableReader&) = delete;
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

8
util/autovector.h
View File

@ -12,6 +12,10 @@
namespace rocksdb {
#ifdef ROCKSDB_LITE
template <class T, size_t kSize = 8>
class autovector : public std::vector<T> {};
#else
// A vector that leverages pre-allocated stack-based array to achieve better
// performance for array with small amount of items.
//
@ -299,5 +303,5 @@ autovector<T, kSize>& autovector<T, kSize>::assign(const autovector& other) {
return *this;
}
} // rocksdb
#endif // ROCKSDB_LITE
} // namespace rocksdb

2
util/blob_store.cc
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "util/blob_store.h"
namespace rocksdb {
@ -266,3 +267,4 @@ Status BlobStore::CreateNewBucket() {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
util/blob_store.h
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include "rocksdb/env.h"
#include "rocksdb/status.h"
@ -159,3 +160,4 @@ class BlobStore {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
util/hash_linklist_rep.cc
View File

@ -4,6 +4,7 @@
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "util/hash_linklist_rep.h"
#include "rocksdb/memtablerep.h"
@ -484,3 +485,4 @@ MemTableRepFactory* NewHashLinkListRepFactory(size_t bucket_count) {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
util/hash_linklist_rep.h
View File

@ -6,6 +6,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include "rocksdb/slice_transform.h"
#include "rocksdb/memtablerep.h"
@ -32,3 +33,4 @@ class HashLinkListRepFactory : public MemTableRepFactory {
};
}
#endif // ROCKSDB_LITE

2
util/hash_skiplist_rep.cc
View File

@ -4,6 +4,7 @@
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "util/hash_skiplist_rep.h"
#include "rocksdb/memtablerep.h"
@ -339,3 +340,4 @@ MemTableRepFactory* NewHashSkipListRepFactory(
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
util/hash_skiplist_rep.h
View File

@ -6,6 +6,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include "rocksdb/slice_transform.h"
#include "rocksdb/memtablerep.h"
@ -39,3 +40,4 @@ class HashSkipListRepFactory : public MemTableRepFactory {
};
}
#endif // ROCKSDB_LITE

2
util/ldb_cmd.cc
View File

@ -3,6 +3,7 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "util/ldb_cmd.h"
#include "db/dbformat.h"
@ -1834,3 +1835,4 @@ void CheckConsistencyCommand::DoCommand() {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
util/ldb_tool.cc
View File

@ -3,6 +3,7 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "rocksdb/ldb_tool.h"
#include "util/ldb_cmd.h"
@ -103,3 +104,4 @@ void LDBTool::Run(int argc, char** argv, Options options) {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

9
util/logging.cc
View File

@ -50,15 +50,6 @@ std::string EscapeString(const Slice& value) {
return r;
}
bool ConsumeChar(Slice* in, char c) {
if (!in->empty() && (*in)[0] == c) {
in->remove_prefix(1);
return true;
} else {
return false;
}
}
bool ConsumeDecimalNumber(Slice* in, uint64_t* val) {
uint64_t v = 0;
int digits = 0;

4
util/logging.h
View File

@ -35,10 +35,6 @@ extern std::string NumberToString(uint64_t num);
// Escapes any non-printable characters found in "value".
extern std::string EscapeString(const Slice& value);
// If *in starts with "c", advances *in past the first character and
// returns true. Otherwise, returns false.
extern bool ConsumeChar(Slice* in, char c);
// Parse a human-readable number from "*in" into *value. On success,
// advances "*in" past the consumed number and sets "*val" to the
// numeric value. Otherwise, returns false and leaves *in in an

2
util/vectorrep.cc
View File

@ -3,6 +3,7 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "rocksdb/memtablerep.h"
#include <unordered_set>
@ -278,3 +279,4 @@ MemTableRep* VectorRepFactory::CreateMemTableRep(
return new VectorRep(compare, arena, count_);
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

4
utilities/backupable/backupable_db.cc
View File

@ -7,6 +7,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "utilities/backupable_db.h"
#include "db/filename.h"
#include "util/coding.h"
@ -1173,3 +1175,5 @@ Status RestoreBackupableDB::DeleteBackup(BackupID backup_id) {
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

4
utilities/geodb/geodb_impl.cc
View File

@ -3,6 +3,8 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#include "utilities/geodb/geodb_impl.h"
#define __STDC_FORMAT_MACROS
@ -425,3 +427,5 @@ void GeoDBImpl::QuadKeyToTile(std::string quadkey, Tile* tile,
}
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

4
utilities/geodb/geodb_impl.h
View File

@ -4,6 +4,8 @@
// of patent rights can be found in the PATENTS file in the same directory.
//
#ifndef ROCKSDB_LITE
#pragma once
#include <algorithm>
#include <cmath>
@ -185,3 +187,5 @@ class GeoDBImpl : public GeoDB {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
utilities/redis/redis_list_exception.h
View File

@ -5,6 +5,7 @@
* Copyright 2013 Facebook
*/
#ifndef ROCKSDB_LITE
#pragma once
#include <exception>
@ -18,3 +19,4 @@ class RedisListException: public std::exception {
};
} // namespace rocksdb
#endif

4
utilities/redis/redis_list_iterator.h
View File

@ -1,3 +1,4 @@
// Copyright 2013 Facebook
/**
* RedisListIterator:
* An abstraction over the "list" concept (e.g.: for redis lists).
@ -34,9 +35,9 @@
* - n bytes of data: the actual data.
*
* @author Deon Nicholas (dnicholas@fb.com)
* Copyright 2013 Facebook
*/
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
@ -306,3 +307,4 @@ class RedisListIterator {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

7
utilities/redis/redis_lists.cc
View File

@ -1,3 +1,4 @@
// Copyright 2013 Facebook
/**
* A (persistent) Redis API built using the rocksdb backend.
* Implements Redis Lists as described on: http://redis.io/commands#list
@ -18,9 +19,9 @@
* wouldn't have to read and re-write the entire list.
*
* @author Deon Nicholas (dnicholas@fb.com)
* Copyright 2013 Facebook
*/
#ifndef ROCKSDB_LITE
#include "redis_lists.h"
#include <iostream>
@ -547,5 +548,5 @@ int RedisLists::Insert(const std::string& key, const std::string& pivot,
return it.Length();
}
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
utilities/redis/redis_lists.h
View File

@ -8,6 +8,7 @@
* Copyright 2013 Facebook
*/
#ifndef ROCKSDB_LITE
#pragma once
#include <string>
@ -104,3 +105,4 @@ class RedisLists {
};
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
utilities/ttl/db_ttl.cc
View File

@ -1,6 +1,7 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#include "utilities/ttl/db_ttl.h"
#include "db/filename.h"
@ -215,3 +216,4 @@ Iterator* DBWithTTL::NewIterator(const ReadOptions& opts,
}
} // namespace rocksdb
#endif // ROCKSDB_LITE

2
utilities/ttl/db_ttl.h
View File

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef ROCKSDB_LITE
#pragma once
#include <deque>
#include <string>
@ -328,3 +329,4 @@ class TtlMergeOperator : public MergeOperator {
};
}
#endif // ROCKSDB_LITE