andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Change DB::GetApproximateSizes for more flexibility needed for MyRocks

Browse Source 
Summary:
Added an option to GetApproximateSizes to exclude file stats, as MyRocks has those counted exactly and we need only stats from memtables.
Closes https://github.com/facebook/rocksdb/pull/1787

Differential Revision: D4441111

Pulled By: IslamAbdelRahman

fbshipit-source-id: c11f4c3
This commit is contained in:
Vitaliy Liptchinsky 2017-01-20 09:26:07 -08:00 committed by Facebook Github Bot
parent 9239103cd4
commit e840213d6e
6 changed files with 67 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -4,6 +4,7 @@
* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
### Bug Fixes
* Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.

10
db/db_impl.cc
View File

@ -5579,7 +5579,9 @@ ColumnFamilyHandle* DBImpl::GetColumnFamilyHandleUnlocked(
void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable) {
uint8_t include_flags) {
assert(include_flags & DB::SizeApproximationFlags::INCLUDE_FILES ||
include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES);
Version* v;
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
auto cfd = cfh->cfd();
@ -5590,8 +5592,10 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
// Convert user_key into a corresponding internal key.
InternalKey k1(range[i].start, kMaxSequenceNumber, kValueTypeForSeek);
InternalKey k2(range[i].limit, kMaxSequenceNumber, kValueTypeForSeek);
sizes[i] = versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
if (include_memtable) {
if (include_flags & DB::SizeApproximationFlags::INCLUDE_FILES) {
sizes[i] = versions_->ApproximateSize(v, k1.Encode(), k2.Encode());
}
if (include_flags & DB::SizeApproximationFlags::INCLUDE_MEMTABLES) {
sizes[i] += sv->mem->ApproximateSize(k1.Encode(), k2.Encode());
sizes[i] += sv->imm->ApproximateSize(k1.Encode(), k2.Encode());
}

3
db/db_impl.h
View File

@ -138,7 +138,8 @@ class DBImpl : public DB {
using DB::GetApproximateSizes;
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable = false) override;
uint8_t include_flags
= INCLUDE_FILES) override;
using DB::CompactRange;
virtual Status CompactRange(const CompactRangeOptions& options,
ColumnFamilyHandle* column_family,

29
db/db_test.cc
View File

@ -1407,17 +1407,19 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
std::string start = Key(50);
std::string end = Key(60);
Range r(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
uint8_t include_both = DB::SizeApproximationFlags::INCLUDE_FILES |
DB::SizeApproximationFlags::INCLUDE_MEMTABLES;
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_GT(size, 6000);
ASSERT_LT(size, 204800);
// Zero if not including mem table
db_->GetApproximateSizes(&r, 1, &size, false);
db_->GetApproximateSizes(&r, 1, &size);
ASSERT_EQ(size, 0);
start = Key(500);
end = Key(600);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
for (int i = 0; i < N; i++) {
@ -1427,13 +1429,13 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
start = Key(500);
end = Key(600);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
start = Key(100);
end = Key(1020);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_GT(size, 6000);
options.max_write_buffer_number = 8;
@ -1456,28 +1458,28 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
start = Key(100);
end = Key(300);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
start = Key(1050);
end = Key(1080);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_GT(size, 6000);
start = Key(2100);
end = Key(2300);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size, true);
db_->GetApproximateSizes(&r, 1, &size, include_both);
ASSERT_EQ(size, 0);
start = Key(1050);
end = Key(1080);
r = Range(start, end);
uint64_t size_with_mt, size_without_mt;
db_->GetApproximateSizes(&r, 1, &size_with_mt, true);
db_->GetApproximateSizes(&r, 1, &size_with_mt, include_both);
ASSERT_GT(size_with_mt, 6000);
db_->GetApproximateSizes(&r, 1, &size_without_mt, false);
db_->GetApproximateSizes(&r, 1, &size_without_mt);
ASSERT_EQ(size_without_mt, 0);
Flush();
@ -1489,8 +1491,8 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
start = Key(1050);
end = Key(1080);
r = Range(start, end);
db_->GetApproximateSizes(&r, 1, &size_with_mt, true);
db_->GetApproximateSizes(&r, 1, &size_without_mt, false);
db_->GetApproximateSizes(&r, 1, &size_with_mt, include_both);
db_->GetApproximateSizes(&r, 1, &size_without_mt);
ASSERT_GT(size_with_mt, size_without_mt);
ASSERT_GT(size_without_mt, 6000);
}
@ -2813,7 +2815,8 @@ class ModelDB : public DB {
using DB::GetApproximateSizes;
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable) override {
uint8_t include_flags
= INCLUDE_FILES) override {
for (int i = 0; i < n; i++) {
sizes[i] = 0;
}

44
include/rocksdb/db.h
View File

@ -586,6 +586,14 @@ class DB {
virtual bool GetAggregatedIntProperty(const Slice& property,
uint64_t* value) = 0;
// Flags for DB::GetSizeApproximation that specify whether memtable
// stats should be included, or file stats approximation or both
enum SizeApproximationFlags : uint8_t {
NONE = 0,
INCLUDE_MEMTABLES = 1,
INCLUDE_FILES = 1 << 1
};
// For each i in [0,n-1], store in "sizes[i]", the approximate
// file system space used by keys in "[range[i].start .. range[i].limit)".
//
@ -593,16 +601,40 @@ class DB {
// if the user data compresses by a factor of ten, the returned
// sizes will be one-tenth the size of the corresponding user data size.
//
// If include_memtable is set to true, then the result will also
// include those recently written data in the mem-tables if
// the mem-table type supports it.
// If include_flags defines whether the returned size should include
// the recently written data in the mem-tables (if
// the mem-table type supports it), data serialized to disk, or both.
// include_flags should be of type DB::SizeApproximationFlags
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable = false) = 0;
uint8_t include_flags
= INCLUDE_FILES) = 0;
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes,
bool include_memtable = false) {
uint8_t include_flags
= INCLUDE_FILES) {
GetApproximateSizes(DefaultColumnFamily(), range, n, sizes,
include_memtable);
include_flags);
}
// Deprecated versions of GetApproximateSizes
ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes(
const Range* range, int n, uint64_t* sizes,
bool include_memtable) {
uint8_t include_flags = SizeApproximationFlags::INCLUDE_FILES;
if (include_memtable) {
include_flags |= SizeApproximationFlags::INCLUDE_MEMTABLES;
}
GetApproximateSizes(DefaultColumnFamily(), range, n, sizes, include_flags);
}
ROCKSDB_DEPRECATED_FUNC virtual void GetApproximateSizes(
ColumnFamilyHandle* column_family,
const Range* range, int n, uint64_t* sizes,
bool include_memtable) {
uint8_t include_flags = SizeApproximationFlags::INCLUDE_FILES;
if (include_memtable) {
include_flags |= SizeApproximationFlags::INCLUDE_MEMTABLES;
}
GetApproximateSizes(column_family, range, n, sizes, include_flags);
}
// Compact the underlying storage for the key range [*begin,*end].

5
include/rocksdb/utilities/stackable_db.h
View File

@ -161,9 +161,10 @@ class StackableDB : public DB {
using DB::GetApproximateSizes;
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
const Range* r, int n, uint64_t* sizes,
bool include_memtable = false) override {
uint8_t include_flags
= INCLUDE_FILES) override {
return db_->GetApproximateSizes(column_family, r, n, sizes,
include_memtable);
include_flags);
}
using DB::CompactRange;