2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2012-11-06 04:18:49 +01:00
|
|
|
|
2019-09-13 22:48:04 +02:00
|
|
|
#include "db/arena_wrapped_db_iter.h"
|
2019-05-31 20:52:59 +02:00
|
|
|
#include "db/db_impl/db_impl_readonly.h"
|
2015-07-14 03:10:31 +02:00
|
|
|
|
|
|
|
#include "db/compacted_db_impl.h"
|
2019-06-01 00:21:36 +02:00
|
|
|
#include "db/db_impl/db_impl.h"
|
2014-09-25 20:14:01 +02:00
|
|
|
#include "db/db_iter.h"
|
2016-11-04 02:40:23 +01:00
|
|
|
#include "db/merge_context.h"
|
2017-04-06 04:02:00 +02:00
|
|
|
#include "monitoring/perf_context_imp.h"
|
2012-11-06 04:18:49 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2012-11-06 04:18:49 +01:00
|
|
|
|
2014-11-26 20:37:59 +01:00
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
2014-09-09 03:46:52 +02:00
|
|
|
DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
|
2014-02-05 22:12:23 +01:00
|
|
|
const std::string& dbname)
|
2014-09-09 03:46:52 +02:00
|
|
|
: DBImpl(db_options, dbname) {
|
2017-03-16 03:22:52 +01:00
|
|
|
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
|
|
|
"Opening the db in read only mode");
|
2016-09-24 01:34:04 +02:00
|
|
|
LogFlush(immutable_db_options_.info_log);
|
2012-11-06 04:18:49 +01:00
|
|
|
}
|
|
|
|
|
2018-04-13 02:55:14 +02:00
|
|
|
DBImplReadOnly::~DBImplReadOnly() {}
|
2012-11-06 04:18:49 +01:00
|
|
|
|
|
|
|
// Implementations of the DB interface
|
2014-09-09 03:46:52 +02:00
|
|
|
Status DBImplReadOnly::Get(const ReadOptions& read_options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
2017-03-13 19:44:50 +01:00
|
|
|
PinnableSlice* pinnable_val) {
|
|
|
|
assert(pinnable_val != nullptr);
|
2018-08-23 07:40:34 +02:00
|
|
|
// TODO: stopwatch DB_GET needed?, perf timer needed?
|
|
|
|
PERF_TIMER_GUARD(get_snapshot_time);
|
2012-11-06 04:18:49 +01:00
|
|
|
Status s;
|
|
|
|
SequenceNumber snapshot = versions_->LastSequence();
|
2014-02-11 02:04:44 +01:00
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
2018-08-01 09:14:43 +02:00
|
|
|
if (tracer_) {
|
|
|
|
InstrumentedMutexLock lock(&trace_mutex_);
|
|
|
|
if (tracer_) {
|
|
|
|
tracer_->Get(column_family, key);
|
|
|
|
}
|
|
|
|
}
|
2014-02-04 00:28:03 +01:00
|
|
|
SuperVersion* super_version = cfd->GetSuperVersion();
|
2013-12-03 03:34:05 +01:00
|
|
|
MergeContext merge_context;
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
2018-10-24 21:29:29 +02:00
|
|
|
SequenceNumber max_covering_tombstone_seq = 0;
|
2012-11-06 04:18:49 +01:00
|
|
|
LookupKey lkey(key, snapshot);
|
2018-08-23 07:40:34 +02:00
|
|
|
PERF_TIMER_STOP(get_snapshot_time);
|
2017-03-13 19:44:50 +01:00
|
|
|
if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
2018-10-24 21:29:29 +02:00
|
|
|
&max_covering_tombstone_seq, read_options)) {
|
2017-03-13 19:44:50 +01:00
|
|
|
pinnable_val->PinSelf();
|
2018-08-23 07:40:34 +02:00
|
|
|
RecordTick(stats_, MEMTABLE_HIT);
|
2013-02-16 00:28:24 +01:00
|
|
|
} else {
|
2014-10-03 02:02:30 +02:00
|
|
|
PERF_TIMER_GUARD(get_from_output_files_time);
|
2017-03-13 19:44:50 +01:00
|
|
|
super_version->current->Get(read_options, lkey, pinnable_val, &s,
|
Use only "local" range tombstones during Get (#4449)
Summary:
Previously, range tombstones were accumulated from every level, which
was necessary if a range tombstone in a higher level covered a key in a lower
level. However, RangeDelAggregator::AddTombstones's complexity is based on
the number of tombstones that are currently stored in it, which is wasteful in
the Get case, where we only need to know the highest sequence number of range
tombstones that cover the key from higher levels, and compute the highest covering
sequence number at the current level. This change introduces this optimization, and
removes the use of RangeDelAggregator from the Get path.
In the benchmark results, the following command was used to initialize the database:
```
./db_bench -db=/dev/shm/5k-rts -use_existing_db=false -benchmarks=filluniquerandom -write_buffer_size=1048576 -compression_type=lz4 -target_file_size_base=1048576 -max_bytes_for_level_base=4194304 -value_size=112 -key_size=16 -block_size=4096 -level_compaction_dynamic_level_bytes=true -num=5000000 -max_background_jobs=12 -benchmark_write_rate_limit=20971520 -range_tombstone_width=100 -writes_per_range_tombstone=100 -max_num_range_tombstones=50000 -bloom_bits=8
```
...and the following command was used to measure read throughput:
```
./db_bench -db=/dev/shm/5k-rts/ -use_existing_db=true -benchmarks=readrandom -disable_auto_compactions=true -num=5000000 -reads=100000 -threads=32
```
The filluniquerandom command was only run once, and the resulting database was used
to measure read performance before and after the PR. Both binaries were compiled with
`DEBUG_LEVEL=0`.
Readrandom results before PR:
```
readrandom : 4.544 micros/op 220090 ops/sec; 16.9 MB/s (63103 of 100000 found)
```
Readrandom results after PR:
```
readrandom : 11.147 micros/op 89707 ops/sec; 6.9 MB/s (63103 of 100000 found)
```
So it's actually slower right now, but this PR paves the way for future optimizations (see #4493).
----
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4449
Differential Revision: D10370575
Pulled By: abhimadan
fbshipit-source-id: 9a2e152be1ef36969055c0e9eb4beb0d96c11f4d
2018-10-24 21:29:29 +02:00
|
|
|
&merge_context, &max_covering_tombstone_seq);
|
2018-08-23 07:40:34 +02:00
|
|
|
RecordTick(stats_, MEMTABLE_MISS);
|
2013-02-16 00:28:24 +01:00
|
|
|
}
|
2018-08-23 07:40:34 +02:00
|
|
|
RecordTick(stats_, NUMBER_KEYS_READ);
|
|
|
|
size_t size = pinnable_val->size();
|
|
|
|
RecordTick(stats_, BYTES_READ, size);
|
2019-02-28 19:14:19 +01:00
|
|
|
RecordInHistogram(stats_, BYTES_PER_READ, size);
|
2018-08-23 07:40:34 +02:00
|
|
|
PERF_COUNTER_ADD(get_read_bytes, size);
|
2012-11-06 04:18:49 +01:00
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-09-09 00:04:34 +02:00
|
|
|
Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family) {
|
|
|
|
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
|
|
|
auto cfd = cfh->cfd();
|
2014-02-04 00:28:03 +01:00
|
|
|
SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
|
|
|
|
SequenceNumber latest_snapshot = versions_->LastSequence();
|
2018-11-29 00:26:56 +01:00
|
|
|
SequenceNumber read_seq =
|
|
|
|
read_options.snapshot != nullptr
|
|
|
|
? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
|
|
|
|
->number_
|
|
|
|
: latest_snapshot;
|
2017-10-10 02:05:34 +02:00
|
|
|
ReadCallback* read_callback = nullptr; // No read callback provided.
|
2014-07-23 22:52:11 +02:00
|
|
|
auto db_iter = NewArenaWrappedDbIterator(
|
2018-05-21 23:33:55 +02:00
|
|
|
env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
|
2018-11-29 00:26:56 +01:00
|
|
|
read_seq,
|
2016-03-01 03:38:03 +01:00
|
|
|
super_version->mutable_cf_options.max_sequential_skip_in_iterations,
|
2017-10-10 02:05:34 +02:00
|
|
|
super_version->version_number, read_callback);
|
2016-11-04 19:53:38 +01:00
|
|
|
auto internal_iter =
|
|
|
|
NewInternalIterator(read_options, cfd, super_version, db_iter->GetArena(),
|
2018-11-29 00:26:56 +01:00
|
|
|
db_iter->GetRangeDelAggregator(), read_seq);
|
2014-07-23 22:52:11 +02:00
|
|
|
db_iter->SetIterUnderDBIter(internal_iter);
|
|
|
|
return db_iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DBImplReadOnly::NewIterators(
|
2014-09-09 00:04:34 +02:00
|
|
|
const ReadOptions& read_options,
|
2014-07-23 22:52:11 +02:00
|
|
|
const std::vector<ColumnFamilyHandle*>& column_families,
|
|
|
|
std::vector<Iterator*>* iterators) {
|
2017-10-10 02:05:34 +02:00
|
|
|
ReadCallback* read_callback = nullptr; // No read callback provided.
|
2014-07-23 22:52:11 +02:00
|
|
|
if (iterators == nullptr) {
|
|
|
|
return Status::InvalidArgument("iterators not allowed to be nullptr");
|
|
|
|
}
|
|
|
|
iterators->clear();
|
|
|
|
iterators->reserve(column_families.size());
|
|
|
|
SequenceNumber latest_snapshot = versions_->LastSequence();
|
2018-11-29 00:26:56 +01:00
|
|
|
SequenceNumber read_seq =
|
|
|
|
read_options.snapshot != nullptr
|
|
|
|
? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
|
|
|
|
->number_
|
|
|
|
: latest_snapshot;
|
2014-07-23 22:52:11 +02:00
|
|
|
|
|
|
|
for (auto cfh : column_families) {
|
2014-10-24 00:34:21 +02:00
|
|
|
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
|
|
|
|
auto* sv = cfd->GetSuperVersion()->Ref();
|
|
|
|
auto* db_iter = NewArenaWrappedDbIterator(
|
2018-11-29 00:26:56 +01:00
|
|
|
env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, read_seq,
|
2016-03-01 03:38:03 +01:00
|
|
|
sv->mutable_cf_options.max_sequential_skip_in_iterations,
|
2017-10-10 02:05:34 +02:00
|
|
|
sv->version_number, read_callback);
|
2016-11-04 19:53:38 +01:00
|
|
|
auto* internal_iter =
|
|
|
|
NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
|
2018-11-29 00:26:56 +01:00
|
|
|
db_iter->GetRangeDelAggregator(), read_seq);
|
2014-07-23 22:52:11 +02:00
|
|
|
db_iter->SetIterUnderDBIter(internal_iter);
|
|
|
|
iterators->push_back(db_iter);
|
|
|
|
}
|
|
|
|
|
|
|
|
return Status::OK();
|
2012-11-06 04:18:49 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
|
2018-03-05 22:08:17 +01:00
|
|
|
DB** dbptr, bool /*error_if_log_file_exist*/) {
|
2013-02-16 00:28:24 +01:00
|
|
|
*dbptr = nullptr;
|
2012-11-06 04:18:49 +01:00
|
|
|
|
2014-09-25 20:14:01 +02:00
|
|
|
// Try to first open DB as fully compacted DB
|
|
|
|
Status s;
|
|
|
|
s = CompactedDBImpl::Open(options, dbname, dbptr);
|
|
|
|
if (s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2014-01-06 22:31:06 +01:00
|
|
|
DBOptions db_options(options);
|
|
|
|
ColumnFamilyOptions cf_options(options);
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
column_families.push_back(
|
2014-04-09 18:56:17 +02:00
|
|
|
ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
|
|
|
|
std::vector<ColumnFamilyHandle*> handles;
|
|
|
|
|
2014-09-25 20:14:01 +02:00
|
|
|
s = DB::OpenForReadOnly(db_options, dbname, column_families, &handles, dbptr);
|
2014-04-09 18:56:17 +02:00
|
|
|
if (s.ok()) {
|
|
|
|
assert(handles.size() == 1);
|
|
|
|
// i can delete the handle since DBImpl is always holding a
|
|
|
|
// reference to default column family
|
|
|
|
delete handles[0];
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DB::OpenForReadOnly(
|
|
|
|
const DBOptions& db_options, const std::string& dbname,
|
|
|
|
const std::vector<ColumnFamilyDescriptor>& column_families,
|
|
|
|
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
|
|
|
|
bool error_if_log_file_exist) {
|
|
|
|
*dbptr = nullptr;
|
|
|
|
handles->clear();
|
2014-02-05 22:12:23 +01:00
|
|
|
|
2017-10-06 03:00:38 +02:00
|
|
|
SuperVersionContext sv_context(/* create_superversion */ true);
|
2014-02-05 22:12:23 +01:00
|
|
|
DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname);
|
|
|
|
impl->mutex_.Lock();
|
2014-01-24 23:30:28 +01:00
|
|
|
Status s = impl->Recover(column_families, true /* read only */,
|
|
|
|
error_if_log_file_exist);
|
2014-04-09 18:56:17 +02:00
|
|
|
if (s.ok()) {
|
|
|
|
// set column family handles
|
|
|
|
for (auto cf : column_families) {
|
|
|
|
auto cfd =
|
|
|
|
impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
|
|
|
|
if (cfd == nullptr) {
|
|
|
|
s = Status::InvalidArgument("Column family not found: ", cf.name);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_));
|
|
|
|
}
|
|
|
|
}
|
2014-02-03 22:13:36 +01:00
|
|
|
if (s.ok()) {
|
2014-02-03 22:44:47 +01:00
|
|
|
for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
|
2017-10-06 03:00:38 +02:00
|
|
|
sv_context.NewSuperVersion();
|
|
|
|
cfd->InstallSuperVersion(&sv_context, &impl->mutex_);
|
2014-02-03 22:44:47 +01:00
|
|
|
}
|
2014-02-03 22:13:36 +01:00
|
|
|
}
|
2012-11-06 04:18:49 +01:00
|
|
|
impl->mutex_.Unlock();
|
2017-10-06 03:00:38 +02:00
|
|
|
sv_context.Clean();
|
2012-11-06 04:18:49 +01:00
|
|
|
if (s.ok()) {
|
|
|
|
*dbptr = impl;
|
2014-11-20 19:49:32 +01:00
|
|
|
for (auto* h : *handles) {
|
|
|
|
impl->NewThreadStatusCfInfo(
|
|
|
|
reinterpret_cast<ColumnFamilyHandleImpl*>(h)->cfd());
|
|
|
|
}
|
2012-11-06 04:18:49 +01:00
|
|
|
} else {
|
2014-04-09 18:56:17 +02:00
|
|
|
for (auto h : *handles) {
|
|
|
|
delete h;
|
|
|
|
}
|
|
|
|
handles->clear();
|
2012-11-06 04:18:49 +01:00
|
|
|
delete impl;
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2018-04-13 02:55:14 +02:00
|
|
|
#else // !ROCKSDB_LITE
|
2014-11-26 20:37:59 +01:00
|
|
|
|
2018-04-13 02:55:14 +02:00
|
|
|
Status DB::OpenForReadOnly(const Options& /*options*/,
|
|
|
|
const std::string& /*dbname*/, DB** /*dbptr*/,
|
|
|
|
bool /*error_if_log_file_exist*/) {
|
2014-11-26 20:37:59 +01:00
|
|
|
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
|
|
|
|
}
|
|
|
|
|
|
|
|
Status DB::OpenForReadOnly(
|
2018-04-13 02:55:14 +02:00
|
|
|
const DBOptions& /*db_options*/, const std::string& /*dbname*/,
|
|
|
|
const std::vector<ColumnFamilyDescriptor>& /*column_families*/,
|
|
|
|
std::vector<ColumnFamilyHandle*>* /*handles*/, DB** /*dbptr*/,
|
|
|
|
bool /*error_if_log_file_exist*/) {
|
2014-11-26 20:37:59 +01:00
|
|
|
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
|
|
|
|
}
|
|
|
|
#endif // !ROCKSDB_LITE
|
2014-04-09 18:56:17 +02:00
|
|
|
|
2018-04-13 02:55:14 +02:00
|
|
|
} // namespace rocksdb
|