2013-10-16 23:59:46 +02:00
|
|
|
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2012-09-24 23:01:01 +02:00
|
|
|
#include <algorithm>
|
2014-03-13 00:40:14 +01:00
|
|
|
#include <iostream>
|
2012-11-26 22:56:45 +01:00
|
|
|
#include <set>
|
2013-11-18 06:58:16 +01:00
|
|
|
#include <unistd.h>
|
2014-02-14 01:28:21 +01:00
|
|
|
#include <unordered_set>
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2014-01-27 22:53:22 +01:00
|
|
|
#include "db/dbformat.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/db_impl.h"
|
|
|
|
#include "db/filename.h"
|
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "db/write_batch_internal.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
2014-01-24 01:32:49 +01:00
|
|
|
#include "rocksdb/db.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/env.h"
|
2014-01-24 01:32:49 +01:00
|
|
|
#include "rocksdb/filter_policy.h"
|
2013-11-18 20:32:54 +01:00
|
|
|
#include "rocksdb/perf_context.h"
|
2014-01-25 01:15:05 +01:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
#include "rocksdb/slice_transform.h"
|
2013-10-29 01:54:09 +01:00
|
|
|
#include "rocksdb/table.h"
|
2014-02-14 01:28:21 +01:00
|
|
|
#include "rocksdb/table_properties.h"
|
2014-01-28 19:35:48 +01:00
|
|
|
#include "table/block_based_table_factory.h"
|
2014-03-13 00:40:14 +01:00
|
|
|
#include "table/plain_table_factory.h"
|
2012-04-17 17:36:46 +02:00
|
|
|
#include "util/hash.h"
|
2014-01-24 01:32:49 +01:00
|
|
|
#include "util/hash_linklist_rep.h"
|
2014-03-13 00:40:14 +01:00
|
|
|
#include "utilities/merge_operators.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/logging.h"
|
2011-09-01 21:08:02 +02:00
|
|
|
#include "util/mutexlock.h"
|
2014-01-24 01:32:49 +01:00
|
|
|
#include "util/statistics.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/testharness.h"
|
2014-03-24 05:49:14 +01:00
|
|
|
#include "util/sync_point.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/testutil.h"
|
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-11-01 18:50:08 +01:00
|
|
|
static bool SnappyCompressionSupported(const CompressionOptions& options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::Snappy_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool ZlibCompressionSupported(const CompressionOptions& options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::Zlib_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool BZip2CompressionSupported(const CompressionOptions& options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::BZip2_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
2012-10-28 07:13:17 +01:00
|
|
|
|
2014-02-08 03:12:30 +01:00
|
|
|
static bool LZ4CompressionSupported(const CompressionOptions &options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::LZ4_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool LZ4HCCompressionSupported(const CompressionOptions &options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::LZ4HC_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
|
|
|
|
|
|
|
static std::string RandomString(Random *rnd, int len) {
|
2011-03-18 23:37:00 +01:00
|
|
|
std::string r;
|
|
|
|
test::RandomString(rnd, len, &r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2012-11-06 21:02:18 +01:00
|
|
|
namespace anon {
|
2012-04-17 17:36:46 +02:00
|
|
|
class AtomicCounter {
|
|
|
|
private:
|
|
|
|
port::Mutex mu_;
|
|
|
|
int count_;
|
|
|
|
public:
|
|
|
|
AtomicCounter() : count_(0) { }
|
|
|
|
void Increment() {
|
|
|
|
MutexLock l(&mu_);
|
|
|
|
count_++;
|
|
|
|
}
|
|
|
|
int Read() {
|
|
|
|
MutexLock l(&mu_);
|
|
|
|
return count_;
|
|
|
|
}
|
|
|
|
void Reset() {
|
|
|
|
MutexLock l(&mu_);
|
|
|
|
count_ = 0;
|
|
|
|
}
|
|
|
|
};
|
2013-07-28 20:53:08 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Special Env used to delay background operations
|
|
|
|
class SpecialEnv : public EnvWrapper {
|
|
|
|
public:
|
2013-03-01 03:04:58 +01:00
|
|
|
// sstable Sync() calls are blocked while this pointer is non-nullptr.
|
2011-06-22 04:36:45 +02:00
|
|
|
port::AtomicPointer delay_sstable_sync_;
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
// Simulate no-space errors while this pointer is non-nullptr.
|
2012-01-25 23:56:52 +01:00
|
|
|
port::AtomicPointer no_space_;
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
// Simulate non-writable file system while this pointer is non-nullptr
|
2012-08-23 01:57:51 +02:00
|
|
|
port::AtomicPointer non_writable_;
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
// Force sync of manifest files to fail while this pointer is non-nullptr
|
2013-01-08 21:00:13 +01:00
|
|
|
port::AtomicPointer manifest_sync_error_;
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
// Force write to manifest files to fail while this pointer is non-nullptr
|
2013-01-08 21:00:13 +01:00
|
|
|
port::AtomicPointer manifest_write_error_;
|
|
|
|
|
2013-10-28 20:36:02 +01:00
|
|
|
// Force write to log files to fail while this pointer is non-nullptr
|
|
|
|
port::AtomicPointer log_write_error_;
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
bool count_random_reads_;
|
2012-11-06 21:02:18 +01:00
|
|
|
anon::AtomicCounter random_read_counter_;
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2014-04-29 19:27:58 +02:00
|
|
|
bool count_sequential_reads_;
|
|
|
|
anon::AtomicCounter sequential_read_counter_;
|
|
|
|
|
2012-11-06 21:02:18 +01:00
|
|
|
anon::AtomicCounter sleep_counter_;
|
2012-08-23 01:57:51 +02:00
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
explicit SpecialEnv(Env* base) : EnvWrapper(base) {
|
2013-03-01 03:04:58 +01:00
|
|
|
delay_sstable_sync_.Release_Store(nullptr);
|
|
|
|
no_space_.Release_Store(nullptr);
|
|
|
|
non_writable_.Release_Store(nullptr);
|
2012-04-17 17:36:46 +02:00
|
|
|
count_random_reads_ = false;
|
2014-04-29 19:27:58 +02:00
|
|
|
count_sequential_reads_ = false;
|
2013-03-01 03:04:58 +01:00
|
|
|
manifest_sync_error_.Release_Store(nullptr);
|
|
|
|
manifest_write_error_.Release_Store(nullptr);
|
2013-10-28 20:36:02 +01:00
|
|
|
log_write_error_.Release_Store(nullptr);
|
2013-01-08 21:00:13 +01:00
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2013-03-15 01:00:04 +01:00
|
|
|
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
|
|
|
|
const EnvOptions& soptions) {
|
2011-06-22 04:36:45 +02:00
|
|
|
class SSTableFile : public WritableFile {
|
|
|
|
private:
|
|
|
|
SpecialEnv* env_;
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<WritableFile> base_;
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
public:
|
2013-01-20 11:07:13 +01:00
|
|
|
SSTableFile(SpecialEnv* env, unique_ptr<WritableFile>&& base)
|
2011-06-22 04:36:45 +02:00
|
|
|
: env_(env),
|
2013-01-20 11:07:13 +01:00
|
|
|
base_(std::move(base)) {
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
2012-01-25 23:56:52 +01:00
|
|
|
Status Append(const Slice& data) {
|
2013-03-01 03:04:58 +01:00
|
|
|
if (env_->no_space_.Acquire_Load() != nullptr) {
|
2012-01-25 23:56:52 +01:00
|
|
|
// Drop writes on the floor
|
|
|
|
return Status::OK();
|
|
|
|
} else {
|
|
|
|
return base_->Append(data);
|
|
|
|
}
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
Status Close() { return base_->Close(); }
|
|
|
|
Status Flush() { return base_->Flush(); }
|
|
|
|
Status Sync() {
|
2013-03-01 03:04:58 +01:00
|
|
|
while (env_->delay_sstable_sync_.Acquire_Load() != nullptr) {
|
2011-06-22 04:36:45 +02:00
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
|
|
|
return base_->Sync();
|
|
|
|
}
|
|
|
|
};
|
2013-01-08 21:00:13 +01:00
|
|
|
class ManifestFile : public WritableFile {
|
|
|
|
private:
|
|
|
|
SpecialEnv* env_;
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<WritableFile> base_;
|
2013-01-08 21:00:13 +01:00
|
|
|
public:
|
2013-01-20 11:07:13 +01:00
|
|
|
ManifestFile(SpecialEnv* env, unique_ptr<WritableFile>&& b)
|
|
|
|
: env_(env), base_(std::move(b)) { }
|
2013-01-08 21:00:13 +01:00
|
|
|
Status Append(const Slice& data) {
|
2013-03-01 03:04:58 +01:00
|
|
|
if (env_->manifest_write_error_.Acquire_Load() != nullptr) {
|
2013-01-08 21:00:13 +01:00
|
|
|
return Status::IOError("simulated writer error");
|
|
|
|
} else {
|
|
|
|
return base_->Append(data);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Status Close() { return base_->Close(); }
|
|
|
|
Status Flush() { return base_->Flush(); }
|
|
|
|
Status Sync() {
|
2013-03-01 03:04:58 +01:00
|
|
|
if (env_->manifest_sync_error_.Acquire_Load() != nullptr) {
|
2013-01-08 21:00:13 +01:00
|
|
|
return Status::IOError("simulated sync error");
|
|
|
|
} else {
|
|
|
|
return base_->Sync();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
2013-10-28 20:36:02 +01:00
|
|
|
class LogFile : public WritableFile {
|
|
|
|
private:
|
|
|
|
SpecialEnv* env_;
|
|
|
|
unique_ptr<WritableFile> base_;
|
|
|
|
public:
|
|
|
|
LogFile(SpecialEnv* env, unique_ptr<WritableFile>&& b)
|
|
|
|
: env_(env), base_(std::move(b)) { }
|
|
|
|
Status Append(const Slice& data) {
|
|
|
|
if (env_->log_write_error_.Acquire_Load() != nullptr) {
|
|
|
|
return Status::IOError("simulated writer error");
|
|
|
|
} else {
|
|
|
|
return base_->Append(data);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Status Close() { return base_->Close(); }
|
|
|
|
Status Flush() { return base_->Flush(); }
|
|
|
|
Status Sync() { return base_->Sync(); }
|
|
|
|
};
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
if (non_writable_.Acquire_Load() != nullptr) {
|
2013-01-08 21:00:13 +01:00
|
|
|
return Status::IOError("simulated write error");
|
|
|
|
}
|
2012-08-23 01:57:51 +02:00
|
|
|
|
2013-03-15 01:00:04 +01:00
|
|
|
Status s = target()->NewWritableFile(f, r, soptions);
|
2011-06-22 04:36:45 +02:00
|
|
|
if (s.ok()) {
|
2013-03-01 03:04:58 +01:00
|
|
|
if (strstr(f.c_str(), ".sst") != nullptr) {
|
2013-01-20 11:07:13 +01:00
|
|
|
r->reset(new SSTableFile(this, std::move(*r)));
|
2013-03-01 03:04:58 +01:00
|
|
|
} else if (strstr(f.c_str(), "MANIFEST") != nullptr) {
|
2013-01-20 11:07:13 +01:00
|
|
|
r->reset(new ManifestFile(this, std::move(*r)));
|
2013-10-28 20:36:02 +01:00
|
|
|
} else if (strstr(f.c_str(), "log") != nullptr) {
|
|
|
|
r->reset(new LogFile(this, std::move(*r)));
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-01-20 11:07:13 +01:00
|
|
|
Status NewRandomAccessFile(const std::string& f,
|
2013-03-15 01:00:04 +01:00
|
|
|
unique_ptr<RandomAccessFile>* r,
|
|
|
|
const EnvOptions& soptions) {
|
2012-04-17 17:36:46 +02:00
|
|
|
class CountingFile : public RandomAccessFile {
|
|
|
|
private:
|
2013-01-20 11:07:13 +01:00
|
|
|
unique_ptr<RandomAccessFile> target_;
|
2012-11-06 21:02:18 +01:00
|
|
|
anon::AtomicCounter* counter_;
|
2012-04-17 17:36:46 +02:00
|
|
|
public:
|
2013-01-20 11:07:13 +01:00
|
|
|
CountingFile(unique_ptr<RandomAccessFile>&& target,
|
|
|
|
anon::AtomicCounter* counter)
|
|
|
|
: target_(std::move(target)), counter_(counter) {
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
|
|
|
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
|
|
|
char* scratch) const {
|
|
|
|
counter_->Increment();
|
|
|
|
return target_->Read(offset, n, result, scratch);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2013-03-15 01:00:04 +01:00
|
|
|
Status s = target()->NewRandomAccessFile(f, r, soptions);
|
2012-04-17 17:36:46 +02:00
|
|
|
if (s.ok() && count_random_reads_) {
|
2013-01-20 11:07:13 +01:00
|
|
|
r->reset(new CountingFile(std::move(*r), &random_read_counter_));
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
2012-08-23 01:57:51 +02:00
|
|
|
|
2014-04-29 19:27:58 +02:00
|
|
|
Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r,
|
|
|
|
const EnvOptions& soptions) {
|
|
|
|
class CountingFile : public SequentialFile {
|
|
|
|
private:
|
|
|
|
unique_ptr<SequentialFile> target_;
|
|
|
|
anon::AtomicCounter* counter_;
|
|
|
|
|
|
|
|
public:
|
|
|
|
CountingFile(unique_ptr<SequentialFile>&& target,
|
|
|
|
anon::AtomicCounter* counter)
|
|
|
|
: target_(std::move(target)), counter_(counter) {}
|
|
|
|
virtual Status Read(size_t n, Slice* result, char* scratch) {
|
|
|
|
counter_->Increment();
|
|
|
|
return target_->Read(n, result, scratch);
|
|
|
|
}
|
|
|
|
virtual Status Skip(uint64_t n) { return target_->Skip(n); }
|
|
|
|
};
|
|
|
|
|
|
|
|
Status s = target()->NewSequentialFile(f, r, soptions);
|
|
|
|
if (s.ok() && count_sequential_reads_) {
|
|
|
|
r->reset(new CountingFile(std::move(*r), &sequential_read_counter_));
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
2012-08-23 01:57:51 +02:00
|
|
|
virtual void SleepForMicroseconds(int micros) {
|
|
|
|
sleep_counter_.Increment();
|
|
|
|
target()->SleepForMicroseconds(micros);
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
};
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
class DBTest {
|
2012-04-17 17:36:46 +02:00
|
|
|
private:
|
|
|
|
const FilterPolicy* filter_policy_;
|
|
|
|
|
2013-08-23 08:10:02 +02:00
|
|
|
protected:
|
2012-04-17 17:36:46 +02:00
|
|
|
// Sequence of option configurations to try
|
|
|
|
enum OptionConfig {
|
2014-07-02 01:27:00 +02:00
|
|
|
kBlockBasedTableWithWholeKeyHashIndex,
|
2014-07-02 02:41:38 +02:00
|
|
|
kDefault,
|
2014-04-10 23:19:43 +02:00
|
|
|
kBlockBasedTableWithPrefixHashIndex,
|
2013-12-20 18:35:24 +01:00
|
|
|
kPlainTableFirstBytePrefix,
|
|
|
|
kPlainTableAllBytesPrefix,
|
2013-08-23 08:10:02 +02:00
|
|
|
kVectorRep,
|
2013-12-27 21:56:27 +01:00
|
|
|
kHashLinkList,
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
kHashCuckoo,
|
2013-03-21 23:59:47 +01:00
|
|
|
kMergePut,
|
2012-04-17 17:36:46 +02:00
|
|
|
kFilter,
|
2012-08-27 08:45:35 +02:00
|
|
|
kUncompressed,
|
2012-06-23 04:30:03 +02:00
|
|
|
kNumLevel_3,
|
2012-09-06 02:44:13 +02:00
|
|
|
kDBLogDir,
|
2013-10-01 23:46:52 +02:00
|
|
|
kWalDir,
|
2013-01-11 02:18:50 +01:00
|
|
|
kManifestFileSize,
|
2013-02-28 23:09:30 +01:00
|
|
|
kCompactOnFlush,
|
2013-03-02 21:56:04 +01:00
|
|
|
kPerfOptions,
|
2013-07-06 03:49:18 +02:00
|
|
|
kDeletesFilterFirst,
|
2013-12-03 21:42:15 +01:00
|
|
|
kHashSkipList,
|
2013-08-02 20:46:47 +02:00
|
|
|
kUniversalCompaction,
|
2013-09-02 08:23:40 +02:00
|
|
|
kCompressedBlockCache,
|
2014-01-07 05:29:17 +01:00
|
|
|
kInfiniteMaxOpenFiles,
|
2014-05-01 20:09:32 +02:00
|
|
|
kxxHashChecksum,
|
2014-05-21 20:43:35 +02:00
|
|
|
kFIFOCompaction,
|
2012-04-17 17:36:46 +02:00
|
|
|
kEnd
|
|
|
|
};
|
|
|
|
int option_config_;
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
public:
|
|
|
|
std::string dbname_;
|
2011-06-22 04:36:45 +02:00
|
|
|
SpecialEnv* env_;
|
2011-03-18 23:37:00 +01:00
|
|
|
DB* db_;
|
2014-02-07 23:47:16 +01:00
|
|
|
std::vector<ColumnFamilyHandle*> handles_;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
Options last_options_;
|
|
|
|
|
2013-08-02 20:46:47 +02:00
|
|
|
// Skip some options, as they may not be applicable to a specific test.
|
|
|
|
// To add more skip constants, use values 4, 8, 16, etc.
|
|
|
|
enum OptionSkip {
|
|
|
|
kNoSkip = 0,
|
|
|
|
kSkipDeletesFilterFirst = 1,
|
2013-10-18 03:33:18 +02:00
|
|
|
kSkipUniversalCompaction = 2,
|
2013-12-20 18:35:24 +01:00
|
|
|
kSkipMergePut = 4,
|
2014-04-10 23:19:43 +02:00
|
|
|
kSkipPlainTable = 8,
|
2014-04-25 21:21:34 +02:00
|
|
|
kSkipHashIndex = 16,
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
kSkipNoSeekToLast = 32,
|
2014-05-21 20:43:35 +02:00
|
|
|
kSkipHashCuckoo = 64,
|
|
|
|
kSkipFIFOCompaction = 128,
|
2013-08-02 20:46:47 +02:00
|
|
|
};
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
DBTest() : option_config_(kDefault),
|
|
|
|
env_(new SpecialEnv(Env::Default())) {
|
|
|
|
filter_policy_ = NewBloomFilterPolicy(10);
|
2011-03-18 23:37:00 +01:00
|
|
|
dbname_ = test::TmpDir() + "/db_test";
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(DestroyDB(dbname_, Options()));
|
2013-03-01 03:04:58 +01:00
|
|
|
db_ = nullptr;
|
2011-03-18 23:37:00 +01:00
|
|
|
Reopen();
|
|
|
|
}
|
|
|
|
|
|
|
|
~DBTest() {
|
2014-02-07 23:47:16 +01:00
|
|
|
Close();
|
2013-10-11 03:02:10 +02:00
|
|
|
ASSERT_OK(DestroyDB(dbname_, Options()));
|
2011-06-22 04:36:45 +02:00
|
|
|
delete env_;
|
2012-04-17 17:36:46 +02:00
|
|
|
delete filter_policy_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Switch to a fresh database with the next option configuration to
|
|
|
|
// test. Return false if there are no more configurations to test.
|
2013-08-02 20:46:47 +02:00
|
|
|
bool ChangeOptions(int skip_mask = kNoSkip) {
|
2013-12-20 18:35:24 +01:00
|
|
|
for(option_config_++; option_config_ < kEnd; option_config_++) {
|
|
|
|
if ((skip_mask & kSkipDeletesFilterFirst) &&
|
|
|
|
option_config_ == kDeletesFilterFirst) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if ((skip_mask & kSkipUniversalCompaction) &&
|
|
|
|
option_config_ == kUniversalCompaction) {
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if ((skip_mask & kSkipMergePut) && option_config_ == kMergePut) {
|
|
|
|
continue;
|
|
|
|
}
|
2014-04-25 21:21:34 +02:00
|
|
|
if ((skip_mask & kSkipNoSeekToLast) &&
|
|
|
|
(option_config_ == kHashLinkList ||
|
|
|
|
option_config_ == kHashSkipList)) {;
|
|
|
|
continue;
|
|
|
|
}
|
2013-12-20 18:35:24 +01:00
|
|
|
if ((skip_mask & kSkipPlainTable)
|
|
|
|
&& (option_config_ == kPlainTableAllBytesPrefix
|
|
|
|
|| option_config_ == kPlainTableFirstBytePrefix)) {
|
|
|
|
continue;
|
|
|
|
}
|
2014-04-10 23:19:43 +02:00
|
|
|
if ((skip_mask & kSkipPlainTable) &&
|
|
|
|
(option_config_ == kBlockBasedTableWithPrefixHashIndex ||
|
|
|
|
option_config_ == kBlockBasedTableWithWholeKeyHashIndex)) {
|
|
|
|
continue;
|
|
|
|
}
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
if ((skip_mask & kSkipHashCuckoo) && (option_config_ == kHashCuckoo)) {
|
|
|
|
continue;
|
|
|
|
}
|
2014-05-21 20:43:35 +02:00
|
|
|
if ((skip_mask & kSkipFIFOCompaction) &&
|
|
|
|
option_config_ == kFIFOCompaction) {
|
|
|
|
continue;
|
|
|
|
}
|
2013-12-20 18:35:24 +01:00
|
|
|
break;
|
2013-10-18 03:33:18 +02:00
|
|
|
}
|
2013-12-20 18:35:24 +01:00
|
|
|
|
2012-08-27 08:45:35 +02:00
|
|
|
if (option_config_ >= kEnd) {
|
2013-10-04 19:21:03 +02:00
|
|
|
Destroy(&last_options_);
|
2012-04-17 17:36:46 +02:00
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
DestroyAndReopen();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Switch between different compaction styles (we have only 2 now).
|
2013-10-01 23:46:52 +02:00
|
|
|
bool ChangeCompactOptions(Options* prev_options = nullptr) {
|
2013-08-08 00:20:41 +02:00
|
|
|
if (option_config_ == kDefault) {
|
|
|
|
option_config_ = kUniversalCompaction;
|
2013-10-01 23:46:52 +02:00
|
|
|
if (prev_options == nullptr) {
|
|
|
|
prev_options = &last_options_;
|
|
|
|
}
|
|
|
|
Destroy(prev_options);
|
|
|
|
TryReopen();
|
2013-08-08 00:20:41 +02:00
|
|
|
return true;
|
|
|
|
} else {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Return the current option configuration.
|
|
|
|
Options CurrentOptions() {
|
|
|
|
Options options;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
return CurrentOptions(options);
|
|
|
|
}
|
|
|
|
|
|
|
|
Options CurrentOptions(const Options& defaultOptions) {
|
|
|
|
// this redudant copy is to minimize code change w/o having lint error.
|
|
|
|
Options options = defaultOptions;
|
2012-04-17 17:36:46 +02:00
|
|
|
switch (option_config_) {
|
2013-12-03 21:42:15 +01:00
|
|
|
case kHashSkipList:
|
2014-03-10 20:56:46 +01:00
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
2014-07-01 00:54:31 +02:00
|
|
|
options.memtable_factory.reset(
|
|
|
|
NewHashSkipListRepFactory(16));
|
2013-08-23 08:10:02 +02:00
|
|
|
break;
|
2013-12-20 18:35:24 +01:00
|
|
|
case kPlainTableFirstBytePrefix:
|
|
|
|
options.table_factory.reset(new PlainTableFactory());
|
2014-03-10 20:56:46 +01:00
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
2013-12-20 18:35:24 +01:00
|
|
|
options.allow_mmap_reads = true;
|
|
|
|
options.max_sequential_skip_in_iterations = 999999;
|
|
|
|
break;
|
|
|
|
case kPlainTableAllBytesPrefix:
|
|
|
|
options.table_factory.reset(new PlainTableFactory());
|
2014-03-10 20:56:46 +01:00
|
|
|
options.prefix_extractor.reset(NewNoopTransform());
|
2013-12-20 18:35:24 +01:00
|
|
|
options.allow_mmap_reads = true;
|
|
|
|
options.max_sequential_skip_in_iterations = 999999;
|
|
|
|
break;
|
2013-03-21 23:59:47 +01:00
|
|
|
case kMergePut:
|
2013-08-20 22:35:28 +02:00
|
|
|
options.merge_operator = MergeOperators::CreatePutOperator();
|
2013-03-21 23:59:47 +01:00
|
|
|
break;
|
2012-04-17 17:36:46 +02:00
|
|
|
case kFilter:
|
|
|
|
options.filter_policy = filter_policy_;
|
|
|
|
break;
|
2012-08-27 08:45:35 +02:00
|
|
|
case kUncompressed:
|
2012-09-06 02:44:13 +02:00
|
|
|
options.compression = kNoCompression;
|
|
|
|
break;
|
2012-06-23 04:30:03 +02:00
|
|
|
case kNumLevel_3:
|
2012-09-06 02:44:13 +02:00
|
|
|
options.num_levels = 3;
|
|
|
|
break;
|
|
|
|
case kDBLogDir:
|
|
|
|
options.db_log_dir = test::TmpDir();
|
|
|
|
break;
|
2013-10-01 23:46:52 +02:00
|
|
|
case kWalDir:
|
2014-07-02 02:41:38 +02:00
|
|
|
options.wal_dir = "/tmp/wal";
|
2013-10-01 23:46:52 +02:00
|
|
|
break;
|
2013-01-11 02:18:50 +01:00
|
|
|
case kManifestFileSize:
|
|
|
|
options.max_manifest_file_size = 50; // 50 bytes
|
2013-02-28 23:09:30 +01:00
|
|
|
case kCompactOnFlush:
|
2013-10-04 19:21:03 +02:00
|
|
|
options.purge_redundant_kvs_while_flush =
|
|
|
|
!options.purge_redundant_kvs_while_flush;
|
2013-03-02 21:56:04 +01:00
|
|
|
break;
|
|
|
|
case kPerfOptions:
|
2013-08-06 00:43:49 +02:00
|
|
|
options.hard_rate_limit = 2.0;
|
|
|
|
options.rate_limit_delay_max_milliseconds = 2;
|
2013-03-02 21:56:04 +01:00
|
|
|
// TODO -- test more options
|
|
|
|
break;
|
2013-07-06 03:49:18 +02:00
|
|
|
case kDeletesFilterFirst:
|
2013-07-13 01:56:52 +02:00
|
|
|
options.filter_deletes = true;
|
2013-07-06 03:49:18 +02:00
|
|
|
break;
|
2013-08-23 08:10:02 +02:00
|
|
|
case kVectorRep:
|
2013-09-25 07:23:19 +02:00
|
|
|
options.memtable_factory.reset(new VectorRepFactory(100));
|
2013-08-23 08:10:02 +02:00
|
|
|
break;
|
2013-12-27 21:56:27 +01:00
|
|
|
case kHashLinkList:
|
2014-03-10 20:56:46 +01:00
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
2014-07-01 20:05:05 +02:00
|
|
|
options.memtable_factory.reset(
|
|
|
|
NewHashLinkListRepFactory(4, 0, 3, true, 4));
|
2013-12-27 21:56:27 +01:00
|
|
|
break;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
case kHashCuckoo:
|
|
|
|
options.memtable_factory.reset(
|
|
|
|
NewHashCuckooRepFactory(options.write_buffer_size));
|
|
|
|
break;
|
2013-08-02 20:46:47 +02:00
|
|
|
case kUniversalCompaction:
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
break;
|
2013-09-02 08:23:40 +02:00
|
|
|
case kCompressedBlockCache:
|
2014-03-29 00:57:04 +01:00
|
|
|
options.allow_mmap_writes = true;
|
2013-09-02 08:23:40 +02:00
|
|
|
options.block_cache_compressed = NewLRUCache(8*1024*1024);
|
|
|
|
break;
|
2014-01-07 05:29:17 +01:00
|
|
|
case kInfiniteMaxOpenFiles:
|
|
|
|
options.max_open_files = -1;
|
|
|
|
break;
|
2014-05-01 20:09:32 +02:00
|
|
|
case kxxHashChecksum: {
|
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
table_options.checksum = kxxHash;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
break;
|
|
|
|
}
|
2014-05-21 20:43:35 +02:00
|
|
|
case kFIFOCompaction: {
|
|
|
|
options.compaction_style = kCompactionStyleFIFO;
|
|
|
|
break;
|
|
|
|
}
|
2014-04-10 23:19:43 +02:00
|
|
|
case kBlockBasedTableWithPrefixHashIndex: {
|
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case kBlockBasedTableWithWholeKeyHashIndex: {
|
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
options.prefix_extractor.reset(NewNoopTransform());
|
|
|
|
break;
|
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return options;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
DBImpl* dbfull() {
|
|
|
|
return reinterpret_cast<DBImpl*>(db_);
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
void CreateColumnFamilies(const std::vector<std::string>& cfs,
|
|
|
|
const ColumnFamilyOptions* options = nullptr) {
|
|
|
|
ColumnFamilyOptions cf_opts;
|
|
|
|
if (options != nullptr) {
|
|
|
|
cf_opts = ColumnFamilyOptions(*options);
|
|
|
|
} else {
|
|
|
|
cf_opts = ColumnFamilyOptions(CurrentOptions());
|
|
|
|
}
|
|
|
|
int cfi = handles_.size();
|
|
|
|
handles_.resize(cfi + cfs.size());
|
|
|
|
for (auto cf : cfs) {
|
|
|
|
ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void CreateAndReopenWithCF(const std::vector<std::string>& cfs,
|
|
|
|
const Options* options = nullptr) {
|
|
|
|
CreateColumnFamilies(cfs, options);
|
|
|
|
std::vector<std::string> cfs_plus_default = cfs;
|
2014-04-09 18:56:17 +02:00
|
|
|
cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies(cfs_plus_default, options);
|
|
|
|
}
|
|
|
|
|
2014-02-15 02:47:53 +01:00
|
|
|
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
|
|
|
|
const std::vector<const Options*>& options) {
|
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
|
|
|
|
const Options* options = nullptr) {
|
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
|
|
|
|
}
|
|
|
|
|
|
|
|
Status TryReopenWithColumnFamilies(
|
|
|
|
const std::vector<std::string>& cfs,
|
|
|
|
const std::vector<const Options*>& options) {
|
|
|
|
Close();
|
|
|
|
ASSERT_EQ(cfs.size(), options.size());
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
for (size_t i = 0; i < cfs.size(); ++i) {
|
|
|
|
column_families.push_back(ColumnFamilyDescriptor(cfs[i], *options[i]));
|
|
|
|
}
|
|
|
|
DBOptions db_opts = DBOptions(*options[0]);
|
|
|
|
return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
|
|
|
|
const Options* options = nullptr) {
|
|
|
|
Close();
|
|
|
|
Options opts = (options == nullptr) ? CurrentOptions() : *options;
|
|
|
|
std::vector<const Options*> v_opts(cfs.size(), &opts);
|
|
|
|
return TryReopenWithColumnFamilies(cfs, v_opts);
|
|
|
|
}
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
void Reopen(Options* options = nullptr) {
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_OK(TryReopen(options));
|
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
void Close() {
|
2014-02-07 23:47:16 +01:00
|
|
|
for (auto h : handles_) {
|
|
|
|
delete h;
|
|
|
|
}
|
|
|
|
handles_.clear();
|
2012-04-17 17:36:46 +02:00
|
|
|
delete db_;
|
2013-03-01 03:04:58 +01:00
|
|
|
db_ = nullptr;
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
void DestroyAndReopen(Options* options = nullptr) {
|
2013-10-04 19:21:03 +02:00
|
|
|
//Destroy using last options
|
|
|
|
Destroy(&last_options_);
|
|
|
|
ASSERT_OK(TryReopen(options));
|
|
|
|
}
|
|
|
|
|
|
|
|
void Destroy(Options* options) {
|
2014-02-07 23:47:16 +01:00
|
|
|
Close();
|
2013-10-11 03:02:10 +02:00
|
|
|
ASSERT_OK(DestroyDB(dbname_, *options));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-03 22:13:36 +01:00
|
|
|
Status ReadOnlyReopen(Options* options) {
|
|
|
|
return DB::OpenForReadOnly(*options, dbname_, &db_);
|
|
|
|
}
|
|
|
|
|
2013-10-01 23:46:52 +02:00
|
|
|
Status TryReopen(Options* options = nullptr) {
|
2014-02-07 23:47:16 +01:00
|
|
|
Close();
|
2011-03-18 23:37:00 +01:00
|
|
|
Options opts;
|
2013-03-01 03:04:58 +01:00
|
|
|
if (options != nullptr) {
|
2011-03-18 23:37:00 +01:00
|
|
|
opts = *options;
|
|
|
|
} else {
|
2012-04-17 17:36:46 +02:00
|
|
|
opts = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
opts.create_if_missing = true;
|
|
|
|
}
|
|
|
|
last_options_ = opts;
|
|
|
|
|
|
|
|
return DB::Open(opts, dbname_, &db_);
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Status Flush(int cf = 0) {
|
|
|
|
if (cf == 0) {
|
|
|
|
return db_->Flush(FlushOptions());
|
|
|
|
} else {
|
|
|
|
return db_->Flush(FlushOptions(), handles_[cf]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
Status Put(const Slice& k, const Slice& v, WriteOptions wo = WriteOptions()) {
|
2013-03-21 23:59:47 +01:00
|
|
|
if (kMergePut == option_config_ ) {
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
return db_->Merge(wo, k, v);
|
2013-03-21 23:59:47 +01:00
|
|
|
} else {
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
return db_->Put(wo, k, v);
|
2013-03-21 23:59:47 +01:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Status Put(int cf, const Slice& k, const Slice& v,
|
|
|
|
WriteOptions wo = WriteOptions()) {
|
|
|
|
if (kMergePut == option_config_) {
|
|
|
|
return db_->Merge(wo, handles_[cf], k, v);
|
|
|
|
} else {
|
|
|
|
return db_->Put(wo, handles_[cf], k, v);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status Delete(const std::string& k) {
|
2011-04-12 21:38:58 +02:00
|
|
|
return db_->Delete(WriteOptions(), k);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Status Delete(int cf, const std::string& k) {
|
|
|
|
return db_->Delete(WriteOptions(), handles_[cf], k);
|
|
|
|
}
|
|
|
|
|
2013-03-01 03:04:58 +01:00
|
|
|
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
2011-03-18 23:37:00 +01:00
|
|
|
ReadOptions options;
|
2013-09-02 08:23:40 +02:00
|
|
|
options.verify_checksums = true;
|
2011-03-18 23:37:00 +01:00
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::string result;
|
|
|
|
Status s = db_->Get(options, k, &result);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
result = "NOT_FOUND";
|
|
|
|
} else if (!s.ok()) {
|
|
|
|
result = s.ToString();
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
std::string Get(int cf, const std::string& k,
|
|
|
|
const Snapshot* snapshot = nullptr) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.verify_checksums = true;
|
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::string result;
|
|
|
|
Status s = db_->Get(options, handles_[cf], k, &result);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
result = "NOT_FOUND";
|
|
|
|
} else if (!s.ok()) {
|
|
|
|
result = s.ToString();
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
// Return a string that contains all key,value pairs in order,
|
|
|
|
// formatted like "(k1->v1)(k2->v2)".
|
2014-02-07 23:47:16 +01:00
|
|
|
std::string Contents(int cf = 0) {
|
2011-10-31 18:22:06 +01:00
|
|
|
std::vector<std::string> forward;
|
|
|
|
std::string result;
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = (cf == 0) ? db_->NewIterator(ReadOptions())
|
|
|
|
: db_->NewIterator(ReadOptions(), handles_[cf]);
|
2011-10-31 18:22:06 +01:00
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
|
|
std::string s = IterStatus(iter);
|
|
|
|
result.push_back('(');
|
|
|
|
result.append(s);
|
|
|
|
result.push_back(')');
|
|
|
|
forward.push_back(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check reverse iteration results are the reverse of forward results
|
2012-11-06 21:02:18 +01:00
|
|
|
unsigned int matched = 0;
|
2011-10-31 18:22:06 +01:00
|
|
|
for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
|
|
|
|
ASSERT_LT(matched, forward.size());
|
|
|
|
ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
|
|
|
|
matched++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(matched, forward.size());
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
std::string AllEntriesFor(const Slice& user_key, int cf = 0) {
|
|
|
|
Iterator* iter;
|
|
|
|
if (cf == 0) {
|
|
|
|
iter = dbfull()->TEST_NewInternalIterator();
|
|
|
|
} else {
|
|
|
|
iter = dbfull()->TEST_NewInternalIterator(handles_[cf]);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
|
|
|
|
iter->Seek(target.Encode());
|
|
|
|
std::string result;
|
|
|
|
if (!iter->status().ok()) {
|
|
|
|
result = iter->status().ToString();
|
|
|
|
} else {
|
|
|
|
result = "[ ";
|
|
|
|
bool first = true;
|
|
|
|
while (iter->Valid()) {
|
2013-11-17 22:52:55 +01:00
|
|
|
ParsedInternalKey ikey(Slice(), 0, kTypeValue);
|
2011-03-18 23:37:00 +01:00
|
|
|
if (!ParseInternalKey(iter->key(), &ikey)) {
|
|
|
|
result += "CORRUPTED";
|
|
|
|
} else {
|
2012-04-17 17:36:46 +02:00
|
|
|
if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) {
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!first) {
|
|
|
|
result += ", ";
|
|
|
|
}
|
|
|
|
first = false;
|
|
|
|
switch (ikey.type) {
|
|
|
|
case kTypeValue:
|
|
|
|
result += iter->value().ToString();
|
|
|
|
break;
|
2013-03-21 23:59:47 +01:00
|
|
|
case kTypeMerge:
|
|
|
|
// keep it the same as kTypeValue for testing kMergePut
|
|
|
|
result += iter->value().ToString();
|
|
|
|
break;
|
2011-03-18 23:37:00 +01:00
|
|
|
case kTypeDeletion:
|
|
|
|
result += "DEL";
|
|
|
|
break;
|
2014-01-14 16:55:16 +01:00
|
|
|
default:
|
2013-08-15 01:32:46 +02:00
|
|
|
assert(false);
|
|
|
|
break;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
if (!first) {
|
|
|
|
result += " ";
|
|
|
|
}
|
|
|
|
result += "]";
|
|
|
|
}
|
|
|
|
delete iter;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
int NumTableFilesAtLevel(int level, int cf = 0) {
|
2011-04-12 21:38:58 +02:00
|
|
|
std::string property;
|
2014-02-07 23:47:16 +01:00
|
|
|
if (cf == 0) {
|
|
|
|
// default cfd
|
|
|
|
ASSERT_TRUE(db_->GetProperty(
|
|
|
|
"rocksdb.num-files-at-level" + NumberToString(level), &property));
|
|
|
|
} else {
|
|
|
|
ASSERT_TRUE(db_->GetProperty(
|
|
|
|
handles_[cf], "rocksdb.num-files-at-level" + NumberToString(level),
|
|
|
|
&property));
|
|
|
|
}
|
2011-04-12 21:38:58 +02:00
|
|
|
return atoi(property.c_str());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
int TotalTableFiles(int cf = 0, int levels = -1) {
|
|
|
|
if (levels == -1) {
|
|
|
|
levels = CurrentOptions().num_levels;
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
int result = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int level = 0; level < levels; level++) {
|
|
|
|
result += NumTableFilesAtLevel(level, cf);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
// Return spread of files per level
|
2014-02-07 23:47:16 +01:00
|
|
|
std::string FilesPerLevel(int cf = 0) {
|
|
|
|
int num_levels =
|
|
|
|
(cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]);
|
2011-10-06 01:30:28 +02:00
|
|
|
std::string result;
|
|
|
|
int last_non_zero_offset = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int level = 0; level < num_levels; level++) {
|
|
|
|
int f = NumTableFilesAtLevel(level, cf);
|
2011-10-06 01:30:28 +02:00
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
|
|
|
|
result += buf;
|
|
|
|
if (f > 0) {
|
|
|
|
last_non_zero_offset = result.size();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
result.resize(last_non_zero_offset);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2012-01-25 23:56:52 +01:00
|
|
|
int CountFiles() {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
env_->GetChildren(dbname_, &files);
|
2013-10-01 23:46:52 +02:00
|
|
|
|
|
|
|
std::vector<std::string> logfiles;
|
|
|
|
if (dbname_ != last_options_.wal_dir) {
|
|
|
|
env_->GetChildren(last_options_.wal_dir, &logfiles);
|
|
|
|
}
|
|
|
|
|
|
|
|
return static_cast<int>(files.size() + logfiles.size());
|
2012-01-25 23:56:52 +01:00
|
|
|
}
|
|
|
|
|
2012-10-29 19:12:24 +01:00
|
|
|
int CountLiveFiles() {
|
2014-02-07 23:47:16 +01:00
|
|
|
std::vector<LiveFileMetaData> metadata;
|
|
|
|
db_->GetLiveFilesMetaData(&metadata);
|
|
|
|
return metadata.size();
|
2012-10-29 19:12:24 +01:00
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
uint64_t Size(const Slice& start, const Slice& limit, int cf = 0) {
|
2011-03-18 23:37:00 +01:00
|
|
|
Range r(start, limit);
|
|
|
|
uint64_t size;
|
2014-02-07 23:47:16 +01:00
|
|
|
if (cf == 0) {
|
|
|
|
db_->GetApproximateSizes(&r, 1, &size);
|
|
|
|
} else {
|
|
|
|
db_->GetApproximateSizes(handles_[1], &r, 1, &size);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
void Compact(int cf, const Slice& start, const Slice& limit) {
|
|
|
|
ASSERT_OK(db_->CompactRange(handles_[cf], &start, &limit));
|
|
|
|
}
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
void Compact(const Slice& start, const Slice& limit) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(db_->CompactRange(&start, &limit));
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Do n memtable compactions, each of which produces an sstable
|
|
|
|
// covering the range [small,large].
|
2014-02-07 23:47:16 +01:00
|
|
|
void MakeTables(int n, const std::string& small, const std::string& large,
|
|
|
|
int cf = 0) {
|
2011-10-06 01:30:28 +02:00
|
|
|
for (int i = 0; i < n; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(cf, small, "begin"));
|
|
|
|
ASSERT_OK(Put(cf, large, "end"));
|
|
|
|
ASSERT_OK(Flush(cf));
|
2011-03-22 19:32:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Prevent pushing of new sstables into deeper levels by adding
|
|
|
|
// tables that cover a specified range to all levels.
|
2014-02-07 23:47:16 +01:00
|
|
|
void FillLevels(const std::string& smallest, const std::string& largest,
|
|
|
|
int cf) {
|
|
|
|
MakeTables(db_->NumberLevels(handles_[cf]), smallest, largest, cf);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
void DumpFileCounts(const char* label) {
|
|
|
|
fprintf(stderr, "---\n%s:\n", label);
|
|
|
|
fprintf(stderr, "maxoverlap: %lld\n",
|
|
|
|
static_cast<long long>(
|
|
|
|
dbfull()->TEST_MaxNextLevelOverlappingBytes()));
|
2012-06-23 04:30:03 +02:00
|
|
|
for (int level = 0; level < db_->NumberLevels(); level++) {
|
2011-03-22 19:32:49 +01:00
|
|
|
int num = NumTableFilesAtLevel(level);
|
|
|
|
if (num > 0) {
|
|
|
|
fprintf(stderr, " level %3d : %d files\n", level, num);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
std::string DumpSSTableList() {
|
|
|
|
std::string property;
|
2013-10-05 07:32:05 +02:00
|
|
|
db_->GetProperty("rocksdb.sstables", &property);
|
2011-10-06 01:30:28 +02:00
|
|
|
return property;
|
|
|
|
}
|
|
|
|
|
2011-03-25 21:27:43 +01:00
|
|
|
std::string IterStatus(Iterator* iter) {
|
|
|
|
std::string result;
|
|
|
|
if (iter->Valid()) {
|
|
|
|
result = iter->key().ToString() + "->" + iter->value().ToString();
|
|
|
|
} else {
|
|
|
|
result = "(invalid)";
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
2013-03-21 23:12:35 +01:00
|
|
|
|
|
|
|
Options OptionsForLogIterTest() {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.WAL_ttl_seconds = 1000;
|
|
|
|
return options;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::unique_ptr<TransactionLogIterator> OpenTransactionLogIter(
|
2013-10-25 04:09:02 +02:00
|
|
|
const SequenceNumber seq) {
|
2013-03-21 23:12:35 +01:00
|
|
|
unique_ptr<TransactionLogIterator> iter;
|
|
|
|
Status status = dbfull()->GetUpdatesSince(seq, &iter);
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(status);
|
2013-03-21 23:12:35 +01:00
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
return std::move(iter);
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string DummyString(size_t len, char c = 'a') {
|
|
|
|
return std::string(len, c);
|
|
|
|
}
|
2013-10-18 03:33:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
void VerifyIterLast(std::string expected_key, int cf = 0) {
|
|
|
|
Iterator* iter;
|
2014-04-25 21:21:34 +02:00
|
|
|
ReadOptions ro;
|
2014-02-07 23:47:16 +01:00
|
|
|
if (cf == 0) {
|
2014-04-25 21:21:34 +02:00
|
|
|
iter = db_->NewIterator(ro);
|
2014-02-07 23:47:16 +01:00
|
|
|
} else {
|
2014-04-25 21:21:34 +02:00
|
|
|
iter = db_->NewIterator(ro, handles_[cf]);
|
2014-02-07 23:47:16 +01:00
|
|
|
}
|
2013-10-18 03:33:18 +02:00
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), expected_key);
|
|
|
|
delete iter;
|
|
|
|
}
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
|
2014-01-14 16:55:16 +01:00
|
|
|
// Used to test InplaceUpdate
|
|
|
|
|
|
|
|
// If previous value is nullptr or delta is > than previous value,
|
|
|
|
// sets newValue with delta
|
|
|
|
// If previous value is not empty,
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
// updates previous value with 'b' string of previous value size - 1.
|
|
|
|
static UpdateStatus
|
|
|
|
updateInPlaceSmallerSize(char* prevValue, uint32_t* prevSize,
|
|
|
|
Slice delta, std::string* newValue) {
|
|
|
|
if (prevValue == nullptr) {
|
2014-01-14 16:55:16 +01:00
|
|
|
*newValue = std::string(delta.size(), 'c');
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
return UpdateStatus::UPDATED;
|
2014-01-14 16:55:16 +01:00
|
|
|
} else {
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
*prevSize = *prevSize - 1;
|
|
|
|
std::string str_b = std::string(*prevSize, 'b');
|
2014-01-14 16:55:16 +01:00
|
|
|
memcpy(prevValue, str_b.c_str(), str_b.size());
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
return UpdateStatus::UPDATED_INPLACE;
|
2014-01-14 16:55:16 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
static UpdateStatus
|
|
|
|
updateInPlaceSmallerVarintSize(char* prevValue, uint32_t* prevSize,
|
|
|
|
Slice delta, std::string* newValue) {
|
|
|
|
if (prevValue == nullptr) {
|
|
|
|
*newValue = std::string(delta.size(), 'c');
|
|
|
|
return UpdateStatus::UPDATED;
|
|
|
|
} else {
|
|
|
|
*prevSize = 1;
|
|
|
|
std::string str_b = std::string(*prevSize, 'b');
|
|
|
|
memcpy(prevValue, str_b.c_str(), str_b.size());
|
|
|
|
return UpdateStatus::UPDATED_INPLACE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static UpdateStatus
|
|
|
|
updateInPlaceLargerSize(char* prevValue, uint32_t* prevSize,
|
|
|
|
Slice delta, std::string* newValue) {
|
|
|
|
*newValue = std::string(delta.size(), 'c');
|
|
|
|
return UpdateStatus::UPDATED;
|
|
|
|
}
|
|
|
|
|
|
|
|
static UpdateStatus
|
|
|
|
updateInPlaceNoAction(char* prevValue, uint32_t* prevSize,
|
|
|
|
Slice delta, std::string* newValue) {
|
|
|
|
return UpdateStatus::UPDATE_FAILED;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Utility method to test InplaceUpdate
|
2014-02-07 23:47:16 +01:00
|
|
|
void validateNumberOfEntries(int numValues, int cf = 0) {
|
|
|
|
Iterator* iter;
|
|
|
|
if (cf != 0) {
|
|
|
|
iter = dbfull()->TEST_NewInternalIterator(handles_[cf]);
|
|
|
|
} else {
|
|
|
|
iter = dbfull()->TEST_NewInternalIterator();
|
|
|
|
}
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(iter->status().ok(), true);
|
|
|
|
int seq = numValues;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
ParsedInternalKey ikey;
|
|
|
|
ikey.sequence = -1;
|
|
|
|
ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
|
|
|
|
|
|
|
|
// checks sequence number for updates
|
|
|
|
ASSERT_EQ(ikey.sequence, (unsigned)seq--);
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
delete iter;
|
|
|
|
ASSERT_EQ(0, seq);
|
2014-01-14 16:55:16 +01:00
|
|
|
}
|
2014-01-24 01:32:49 +01:00
|
|
|
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
void CopyFile(const std::string& source, const std::string& destination,
|
|
|
|
uint64_t size = 0) {
|
|
|
|
const EnvOptions soptions;
|
|
|
|
unique_ptr<SequentialFile> srcfile;
|
|
|
|
ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions));
|
|
|
|
unique_ptr<WritableFile> destfile;
|
|
|
|
ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions));
|
|
|
|
|
|
|
|
if (size == 0) {
|
|
|
|
// default argument means copy everything
|
|
|
|
ASSERT_OK(env_->GetFileSize(source, &size));
|
|
|
|
}
|
|
|
|
|
|
|
|
char buffer[4096];
|
|
|
|
Slice slice;
|
|
|
|
while (size > 0) {
|
|
|
|
uint64_t one = std::min(uint64_t(sizeof(buffer)), size);
|
|
|
|
ASSERT_OK(srcfile->Read(one, &slice, buffer));
|
|
|
|
ASSERT_OK(destfile->Append(slice));
|
|
|
|
size -= slice.size();
|
|
|
|
}
|
|
|
|
ASSERT_OK(destfile->Close());
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
2013-11-17 08:44:39 +01:00
|
|
|
static std::string Key(int i) {
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "key%06d", i);
|
|
|
|
return std::string(buf);
|
|
|
|
}
|
|
|
|
|
2014-01-17 21:46:06 +01:00
|
|
|
static long TestGetTickerCount(const Options& options, Tickers ticker_type) {
|
|
|
|
return options.statistics->getTickerCount(ticker_type);
|
|
|
|
}
|
|
|
|
|
2014-02-14 01:28:21 +01:00
|
|
|
// A helper function that ensures the table properties returned in
|
|
|
|
// `GetPropertiesOfAllTablesTest` is correct.
|
|
|
|
// This test assumes entries size is differnt for each of the tables.
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2014-02-14 01:28:21 +01:00
|
|
|
void VerifyTableProperties(DB* db, uint64_t expected_entries_size) {
|
|
|
|
TablePropertiesCollection props;
|
|
|
|
ASSERT_OK(db->GetPropertiesOfAllTables(&props));
|
|
|
|
|
2014-02-15 01:18:55 +01:00
|
|
|
ASSERT_EQ(4U, props.size());
|
2014-02-14 01:28:21 +01:00
|
|
|
std::unordered_set<uint64_t> unique_entries;
|
|
|
|
|
|
|
|
// Indirect test
|
|
|
|
uint64_t sum = 0;
|
|
|
|
for (const auto& item : props) {
|
|
|
|
unique_entries.insert(item.second->num_entries);
|
|
|
|
sum += item.second->num_entries;
|
|
|
|
}
|
|
|
|
|
|
|
|
ASSERT_EQ(props.size(), unique_entries.size());
|
|
|
|
ASSERT_EQ(expected_entries_size, sum);
|
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2014-03-13 00:40:14 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, Empty) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-12-27 21:23:17 +01:00
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000; // Small write buffer
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-04-23 02:17:33 +02:00
|
|
|
std::string num;
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ("0", num);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
2014-04-23 02:17:33 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ("1", num);
|
2013-12-27 21:23:17 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
|
|
|
|
Put(1, "k1", std::string(100000, 'x')); // Fill memtable
|
2014-04-23 02:17:33 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ("2", num);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "k2", std::string(100000, 'y')); // Trigger compaction
|
2014-04-23 02:17:33 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ("1", num);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
2013-12-27 21:23:17 +01:00
|
|
|
env_->delay_sstable_sync_.Release_Store(nullptr); // Release sync calls
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-03 22:13:36 +01:00
|
|
|
TEST(DBTest, ReadOnlyDB) {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_OK(Put("bar", "v2"));
|
|
|
|
ASSERT_OK(Put("foo", "v3"));
|
|
|
|
Close();
|
|
|
|
|
|
|
|
Options options;
|
|
|
|
ASSERT_OK(ReadOnlyReopen(&options));
|
|
|
|
ASSERT_EQ("v3", Get("foo"));
|
|
|
|
ASSERT_EQ("v2", Get("bar"));
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
int count = 0;
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
++count;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(count, 2);
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2013-11-13 07:46:51 +01:00
|
|
|
// Make sure that when options.block_cache is set, after a new table is
|
|
|
|
// created its index/filter blocks are added to block cache.
|
|
|
|
TEST(DBTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
std::unique_ptr<const FilterPolicy> filter_policy(NewBloomFilterPolicy(20));
|
|
|
|
options.filter_policy = filter_policy.get();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.statistics = rocksdb::CreateDBStatistics();
|
2014-01-24 19:57:15 +01:00
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
table_options.cache_index_and_filter_blocks = true;
|
|
|
|
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-11-13 07:46:51 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", "val"));
|
|
|
|
// Create a new table.
|
|
|
|
ASSERT_OK(Flush(1));
|
2013-11-13 07:46:51 +01:00
|
|
|
|
|
|
|
// index/filter blocks added to block cache right after table creation.
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
|
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
2013-11-13 07:46:51 +01:00
|
|
|
ASSERT_EQ(2, /* only index/filter were added */
|
2014-01-17 21:46:06 +01:00
|
|
|
TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
|
|
|
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
|
2013-11-13 07:46:51 +01:00
|
|
|
|
|
|
|
// Make sure filter block is in cache.
|
|
|
|
std::string value;
|
|
|
|
ReadOptions ropt;
|
2014-02-07 23:47:16 +01:00
|
|
|
db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
|
2013-11-13 07:46:51 +01:00
|
|
|
|
|
|
|
// Miss count should remain the same.
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
2013-11-13 07:46:51 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
|
|
|
ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
2013-11-13 07:46:51 +01:00
|
|
|
|
|
|
|
// Make sure index block is in cache.
|
2014-01-17 21:46:06 +01:00
|
|
|
auto index_block_hit = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
|
2014-02-07 23:47:16 +01:00
|
|
|
value = Get(1, "key");
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
2013-11-13 07:46:51 +01:00
|
|
|
ASSERT_EQ(index_block_hit + 1,
|
2014-01-17 21:46:06 +01:00
|
|
|
TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
2013-11-13 07:46:51 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
value = Get(1, "key");
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
|
2013-11-13 07:46:51 +01:00
|
|
|
ASSERT_EQ(index_block_hit + 2,
|
2014-01-17 21:46:06 +01:00
|
|
|
TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
2013-11-13 07:46:51 +01:00
|
|
|
}
|
|
|
|
|
2014-02-14 01:28:21 +01:00
|
|
|
TEST(DBTest, GetPropertiesOfAllTablesTest) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
Reopen(&options);
|
|
|
|
// Create 4 tables
|
|
|
|
for (int table = 0; table < 4; ++table) {
|
|
|
|
for (int i = 0; i < 10 + table; ++i) {
|
|
|
|
db_->Put(WriteOptions(), std::to_string(table * 100 + i), "val");
|
|
|
|
}
|
|
|
|
db_->Flush(FlushOptions());
|
|
|
|
}
|
|
|
|
|
|
|
|
// 1. Read table properties directly from file
|
|
|
|
Reopen(&options);
|
|
|
|
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
|
|
|
|
|
|
|
// 2. Put two tables to table cache and
|
|
|
|
Reopen(&options);
|
|
|
|
// fetch key from 1st and 2nd table, which will internally place that table to
|
|
|
|
// the table cache.
|
|
|
|
for (int i = 0; i < 2; ++i) {
|
|
|
|
Get(std::to_string(i * 100 + 0));
|
|
|
|
}
|
|
|
|
|
|
|
|
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
|
|
|
|
|
|
|
// 3. Put all tables to table cache
|
|
|
|
Reopen(&options);
|
|
|
|
// fetch key from 1st and 2nd table, which will internally place that table to
|
|
|
|
// the table cache.
|
|
|
|
for (int i = 0; i < 4; ++i) {
|
|
|
|
Get(std::to_string(i * 100 + 0));
|
|
|
|
}
|
|
|
|
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
|
|
|
|
}
|
|
|
|
|
2013-01-24 19:54:26 +01:00
|
|
|
TEST(DBTest, LevelLimitReopen) {
|
|
|
|
Options options = CurrentOptions();
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-01-24 19:54:26 +01:00
|
|
|
|
|
|
|
const std::string value(1024 * 1024, ' ');
|
|
|
|
int i = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
while (NumTableFilesAtLevel(2, 1) == 0) {
|
|
|
|
ASSERT_OK(Put(1, Key(i++), value));
|
2013-01-24 19:54:26 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
options.num_levels = 1;
|
2013-05-23 19:56:36 +02:00
|
|
|
options.max_bytes_for_level_multiplier_additional.resize(1, 1);
|
2014-02-07 23:47:16 +01:00
|
|
|
Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2014-01-15 00:27:09 +01:00
|
|
|
ASSERT_EQ(s.IsInvalidArgument(), true);
|
2013-01-24 19:54:26 +01:00
|
|
|
ASSERT_EQ(s.ToString(),
|
2014-01-15 00:27:09 +01:00
|
|
|
"Invalid argument: db has more levels than options.num_levels");
|
2013-01-24 19:54:26 +01:00
|
|
|
|
|
|
|
options.num_levels = 10;
|
2013-05-23 19:56:36 +02:00
|
|
|
options.max_bytes_for_level_multiplier_additional.resize(10, 1);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, &options));
|
2013-01-24 19:54:26 +01:00
|
|
|
}
|
|
|
|
|
2013-01-15 23:05:42 +01:00
|
|
|
TEST(DBTest, Preallocation) {
|
|
|
|
const std::string src = dbname_ + "/alloc_test";
|
|
|
|
unique_ptr<WritableFile> srcfile;
|
2013-06-08 00:35:17 +02:00
|
|
|
const EnvOptions soptions;
|
2013-03-15 01:00:04 +01:00
|
|
|
ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions));
|
2013-01-15 23:05:42 +01:00
|
|
|
srcfile->SetPreallocationBlockSize(1024 * 1024);
|
|
|
|
|
|
|
|
// No writes should mean no preallocation
|
|
|
|
size_t block_size, last_allocated_block;
|
|
|
|
srcfile->GetPreallocationStatus(&block_size, &last_allocated_block);
|
|
|
|
ASSERT_EQ(last_allocated_block, 0UL);
|
|
|
|
|
|
|
|
// Small write should preallocate one block
|
|
|
|
srcfile->Append("test");
|
|
|
|
srcfile->GetPreallocationStatus(&block_size, &last_allocated_block);
|
|
|
|
ASSERT_EQ(last_allocated_block, 1UL);
|
|
|
|
|
|
|
|
// Write an entire preallocation block, make sure we increased by two.
|
|
|
|
std::string buf(block_size, ' ');
|
|
|
|
srcfile->Append(buf);
|
|
|
|
srcfile->GetPreallocationStatus(&block_size, &last_allocated_block);
|
|
|
|
ASSERT_EQ(last_allocated_block, 2UL);
|
|
|
|
|
|
|
|
// Write five more blocks at once, ensure we're where we need to be.
|
|
|
|
buf = std::string(block_size * 5, ' ');
|
|
|
|
srcfile->Append(buf);
|
|
|
|
srcfile->GetPreallocationStatus(&block_size, &last_allocated_block);
|
|
|
|
ASSERT_EQ(last_allocated_block, 7UL);
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, PutDeleteGet) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v2"));
|
|
|
|
ASSERT_EQ("v2", Get(1, "foo"));
|
|
|
|
ASSERT_OK(Delete(1, "foo"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-11-13 07:46:51 +01:00
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
TEST(DBTest, GetFromImmutableLayer) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2012-04-17 17:36:46 +02:00
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000; // Small write buffer
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "k1", std::string(100000, 'x')); // Fill memtable
|
|
|
|
Put(1, "k2", std::string(100000, 'y')); // Trigger flush
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
|
2013-03-01 03:04:58 +01:00
|
|
|
env_->delay_sstable_sync_.Release_Store(nullptr); // Release sync calls
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetFromVersions) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetSnapshot) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-04-17 17:36:46 +02:00
|
|
|
// Try with both a short key and a long key
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, key, "v1"));
|
2012-04-17 17:36:46 +02:00
|
|
|
const Snapshot* s1 = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, key, "v2"));
|
|
|
|
ASSERT_EQ("v2", Get(1, key));
|
|
|
|
ASSERT_EQ("v1", Get(1, key, s1));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ("v2", Get(1, key));
|
|
|
|
ASSERT_EQ("v1", Get(1, key, s1));
|
2012-04-17 17:36:46 +02:00
|
|
|
db_->ReleaseSnapshot(s1);
|
|
|
|
}
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// skip as HashCuckooRep does not support snapshot
|
|
|
|
} while (ChangeOptions(kSkipHashCuckoo));
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetLevel0Ordering) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-04-17 17:36:46 +02:00
|
|
|
// Check that we process level-0 files in correct order. The code
|
|
|
|
// below generates two level-0 files where the earlier one comes
|
|
|
|
// before the later one in the level-0 file list since the earlier
|
|
|
|
// one has a smaller "smallest" key.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "bar", "b"));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v2"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ("v2", Get(1, "foo"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetOrderedByLevels) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
Compact(1, "a", "z");
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v2"));
|
|
|
|
ASSERT_EQ("v2", Get(1, "foo"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ("v2", Get(1, "foo"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetPicksCorrectFile) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-04-17 17:36:46 +02:00
|
|
|
// Arrange to have multiple files in a non-level-0 level.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "va"));
|
|
|
|
Compact(1, "a", "b");
|
|
|
|
ASSERT_OK(Put(1, "x", "vx"));
|
|
|
|
Compact(1, "x", "y");
|
|
|
|
ASSERT_OK(Put(1, "f", "vf"));
|
|
|
|
Compact(1, "f", "g");
|
|
|
|
ASSERT_EQ("va", Get(1, "a"));
|
|
|
|
ASSERT_EQ("vf", Get(1, "f"));
|
|
|
|
ASSERT_EQ("vx", Get(1, "x"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-09-01 21:08:02 +02:00
|
|
|
TEST(DBTest, GetEncountersEmptyLevel) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-04-17 17:36:46 +02:00
|
|
|
// Arrange for the following to happen:
|
|
|
|
// * sstable A in level 0
|
|
|
|
// * nothing in level 1
|
|
|
|
// * sstable B in level 2
|
|
|
|
// Then do enough Get() calls to arrange for an automatic compaction
|
|
|
|
// of sstable A. A bug would cause the compaction to be marked as
|
|
|
|
// occuring at level 1 (instead of the correct level 0).
|
|
|
|
|
|
|
|
// Step 1: First place sstables in levels 0 and 2
|
|
|
|
int compaction_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
while (NumTableFilesAtLevel(0, 1) == 0 || NumTableFilesAtLevel(2, 1) == 0) {
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
|
|
|
|
compaction_count++;
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "a", "begin");
|
|
|
|
Put(1, "z", "end");
|
|
|
|
ASSERT_OK(Flush(1));
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2011-09-01 21:08:02 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Step 2: clear level 1 if necessary.
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(2, 1), 1);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2012-08-27 08:45:35 +02:00
|
|
|
// Step 3: read a bunch of times
|
|
|
|
for (int i = 0; i < 1000; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, "missing"));
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2012-08-27 08:45:35 +02:00
|
|
|
|
|
|
|
// Step 4: Wait for compaction to finish
|
|
|
|
env_->SleepForMicroseconds(1000000);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1); // XXX
|
2014-05-21 20:43:35 +02:00
|
|
|
} while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction));
|
2011-09-01 21:08:02 +02:00
|
|
|
}
|
|
|
|
|
2013-07-26 21:57:01 +02:00
|
|
|
// KeyMayExist can lead to a few false positives, but not false negatives.
|
|
|
|
// To make test deterministic, use a much larger number of bits per key-20 than
|
|
|
|
// bits in the key, so that false positives are eliminated
|
2013-07-06 03:49:18 +02:00
|
|
|
TEST(DBTest, KeyMayExist) {
|
|
|
|
do {
|
2013-07-26 21:57:01 +02:00
|
|
|
ReadOptions ropts;
|
|
|
|
std::string value;
|
2013-07-06 03:49:18 +02:00
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.filter_policy = NewBloomFilterPolicy(20);
|
2013-10-04 06:49:15 +02:00
|
|
|
options.statistics = rocksdb::CreateDBStatistics();
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-07-06 03:49:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value));
|
2013-07-06 03:49:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "b"));
|
2013-07-26 21:57:01 +02:00
|
|
|
bool value_found = false;
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(
|
|
|
|
db_->KeyMayExist(ropts, handles_[1], "a", &value, &value_found));
|
2013-07-26 21:57:01 +02:00
|
|
|
ASSERT_TRUE(value_found);
|
|
|
|
ASSERT_EQ("b", value);
|
2013-07-06 03:49:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2013-07-26 21:57:01 +02:00
|
|
|
value.clear();
|
2013-08-25 07:48:51 +02:00
|
|
|
|
2014-01-17 21:46:06 +01:00
|
|
|
long numopen = TestGetTickerCount(options, NO_FILE_OPENS);
|
|
|
|
long cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(
|
|
|
|
db_->KeyMayExist(ropts, handles_[1], "a", &value, &value_found));
|
2013-08-25 22:40:02 +02:00
|
|
|
ASSERT_TRUE(!value_found);
|
2013-08-25 07:48:51 +02:00
|
|
|
// assert that no new files were opened and no new blocks were
|
|
|
|
// read into block cache.
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
|
|
|
|
ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
2013-07-06 03:49:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "a"));
|
2013-08-25 07:48:51 +02:00
|
|
|
|
2014-01-17 21:46:06 +01:00
|
|
|
numopen = TestGetTickerCount(options, NO_FILE_OPENS);
|
|
|
|
cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value));
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
|
|
|
|
ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
2013-07-06 03:49:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
db_->CompactRange(handles_[1], nullptr, nullptr);
|
2013-08-25 07:48:51 +02:00
|
|
|
|
2014-01-17 21:46:06 +01:00
|
|
|
numopen = TestGetTickerCount(options, NO_FILE_OPENS);
|
|
|
|
cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "a", &value));
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
|
|
|
|
ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
2013-07-06 03:49:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "c"));
|
2013-08-25 07:48:51 +02:00
|
|
|
|
2014-01-17 21:46:06 +01:00
|
|
|
numopen = TestGetTickerCount(options, NO_FILE_OPENS);
|
|
|
|
cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(!db_->KeyMayExist(ropts, handles_[1], "c", &value));
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
|
|
|
|
ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
2013-07-11 23:05:31 +02:00
|
|
|
|
|
|
|
delete options.filter_policy;
|
2013-12-20 18:35:24 +01:00
|
|
|
|
|
|
|
// KeyMayExist function only checks data in block caches, which is not used
|
|
|
|
// by plain table format.
|
2014-05-21 20:43:35 +02:00
|
|
|
} while (
|
|
|
|
ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
|
2013-07-06 03:49:18 +02:00
|
|
|
}
|
|
|
|
|
2013-08-25 07:48:51 +02:00
|
|
|
TEST(DBTest, NonBlockingIteration) {
|
|
|
|
do {
|
|
|
|
ReadOptions non_blocking_opts, regular_opts;
|
|
|
|
Options options = CurrentOptions();
|
2013-10-04 06:49:15 +02:00
|
|
|
options.statistics = rocksdb::CreateDBStatistics();
|
2013-08-25 07:48:51 +02:00
|
|
|
non_blocking_opts.read_tier = kBlockCacheTier;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-25 07:48:51 +02:00
|
|
|
// write one kv to the database.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "b"));
|
2013-08-25 07:48:51 +02:00
|
|
|
|
|
|
|
// scan using non-blocking iterator. We should find it because
|
|
|
|
// it is in memtable.
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = db_->NewIterator(non_blocking_opts, handles_[1]);
|
2013-08-25 07:48:51 +02:00
|
|
|
int count = 0;
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(iter->status());
|
2013-08-25 07:48:51 +02:00
|
|
|
count++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(count, 1);
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// flush memtable to storage. Now, the key should not be in the
|
|
|
|
// memtable neither in the block cache.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2013-08-25 07:48:51 +02:00
|
|
|
|
|
|
|
// verify that a non-blocking iterator does not find any
|
|
|
|
// kvs. Neither does it do any IOs to storage.
|
2014-01-17 21:46:06 +01:00
|
|
|
long numopen = TestGetTickerCount(options, NO_FILE_OPENS);
|
|
|
|
long cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
|
2014-02-07 23:47:16 +01:00
|
|
|
iter = db_->NewIterator(non_blocking_opts, handles_[1]);
|
2013-08-25 07:48:51 +02:00
|
|
|
count = 0;
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(count, 0);
|
|
|
|
ASSERT_TRUE(iter->status().IsIncomplete());
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
|
|
|
|
ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
2013-08-25 07:48:51 +02:00
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// read in the specified block via a regular get
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Get(1, "a"), "b");
|
2013-08-25 07:48:51 +02:00
|
|
|
|
|
|
|
// verify that we can find it via a non-blocking scan
|
2014-01-17 21:46:06 +01:00
|
|
|
numopen = TestGetTickerCount(options, NO_FILE_OPENS);
|
|
|
|
cache_added = TestGetTickerCount(options, BLOCK_CACHE_ADD);
|
2014-02-07 23:47:16 +01:00
|
|
|
iter = db_->NewIterator(non_blocking_opts, handles_[1]);
|
2013-08-25 07:48:51 +02:00
|
|
|
count = 0;
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(iter->status());
|
2013-08-25 07:48:51 +02:00
|
|
|
count++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(count, 1);
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(numopen, TestGetTickerCount(options, NO_FILE_OPENS));
|
|
|
|
ASSERT_EQ(cache_added, TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
2013-08-25 07:48:51 +02:00
|
|
|
delete iter;
|
|
|
|
|
2013-12-20 18:35:24 +01:00
|
|
|
// This test verifies block cache behaviors, which is not used by plain
|
|
|
|
// table format.
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// Exclude kHashCuckoo as it does not support iteration currently
|
|
|
|
} while (ChangeOptions(kSkipPlainTable | kSkipNoSeekToLast |
|
|
|
|
kSkipHashCuckoo));
|
2013-08-25 07:48:51 +02:00
|
|
|
}
|
|
|
|
|
2013-07-13 01:56:52 +02:00
|
|
|
// A delete is skipped for key if KeyMayExist(key) returns False
|
|
|
|
// Tests Writebatch consistency and proper delete behaviour
|
|
|
|
TEST(DBTest, FilterDeletes) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.filter_policy = NewBloomFilterPolicy(20);
|
|
|
|
options.filter_deletes = true;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-08 00:20:41 +02:00
|
|
|
WriteBatch batch;
|
|
|
|
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Delete(handles_[1], "a");
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->Write(WriteOptions(), &batch);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("a", 1), "[ ]"); // Delete skipped
|
2013-08-08 00:20:41 +02:00
|
|
|
batch.Clear();
|
|
|
|
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Put(handles_[1], "a", "b");
|
|
|
|
batch.Delete(handles_[1], "a");
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->Write(WriteOptions(), &batch);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Get(1, "a"), "NOT_FOUND");
|
|
|
|
ASSERT_EQ(AllEntriesFor("a", 1), "[ DEL, b ]"); // Delete issued
|
2013-08-08 00:20:41 +02:00
|
|
|
batch.Clear();
|
|
|
|
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Delete(handles_[1], "c");
|
|
|
|
batch.Put(handles_[1], "c", "d");
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->Write(WriteOptions(), &batch);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Get(1, "c"), "d");
|
|
|
|
ASSERT_EQ(AllEntriesFor("c", 1), "[ d ]"); // Delete skipped
|
2013-08-08 00:20:41 +02:00
|
|
|
batch.Clear();
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1)); // A stray Flush
|
2013-08-08 00:20:41 +02:00
|
|
|
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Delete(handles_[1], "c");
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->Write(WriteOptions(), &batch);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("c", 1), "[ DEL, d ]"); // Delete issued
|
2013-08-08 00:20:41 +02:00
|
|
|
batch.Clear();
|
2013-07-13 01:56:52 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
delete options.filter_policy;
|
|
|
|
} while (ChangeCompactOptions());
|
2013-07-13 01:56:52 +02:00
|
|
|
}
|
|
|
|
|
2014-03-05 18:13:07 +01:00
|
|
|
|
|
|
|
TEST(DBTest, IterSeekBeforePrev) {
|
|
|
|
ASSERT_OK(Put("a", "b"));
|
|
|
|
ASSERT_OK(Put("c", "d"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(Put("0", "f"));
|
|
|
|
ASSERT_OK(Put("1", "h"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(Put("2", "j"));
|
|
|
|
auto iter = db_->NewIterator(ReadOptions());
|
|
|
|
iter->Seek(Slice("c"));
|
|
|
|
iter->Prev();
|
|
|
|
iter->Seek(Slice("a"));
|
|
|
|
iter->Prev();
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2014-04-01 23:45:30 +02:00
|
|
|
std::string MakeLongKey(size_t length, char c) {
|
|
|
|
return std::string(length, c);
|
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2014-04-01 23:45:30 +02:00
|
|
|
|
|
|
|
TEST(DBTest, IterLongKeys) {
|
|
|
|
ASSERT_OK(Put(MakeLongKey(20, 0), "0"));
|
|
|
|
ASSERT_OK(Put(MakeLongKey(32, 2), "2"));
|
|
|
|
ASSERT_OK(Put("a", "b"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(Put(MakeLongKey(50, 1), "1"));
|
|
|
|
ASSERT_OK(Put(MakeLongKey(127, 3), "3"));
|
|
|
|
ASSERT_OK(Put(MakeLongKey(64, 4), "4"));
|
|
|
|
auto iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
// Create a key that needs to be skipped for Seq too new
|
|
|
|
iter->Seek(MakeLongKey(20, 0));
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(20, 0) + "->0");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(64, 4) + "->4");
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
iter = db_->NewIterator(ReadOptions());
|
|
|
|
iter->Seek(MakeLongKey(50, 1));
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(50, 1) + "->1");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(32, 2) + "->2");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), MakeLongKey(127, 3) + "->3");
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2014-03-15 00:06:38 +01:00
|
|
|
TEST(DBTest, IterNextWithNewerSeq) {
|
|
|
|
ASSERT_OK(Put("0", "0"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(Put("a", "b"));
|
|
|
|
ASSERT_OK(Put("c", "d"));
|
|
|
|
ASSERT_OK(Put("d", "e"));
|
|
|
|
auto iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
// Create a key that needs to be skipped for Seq too new
|
|
|
|
for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1;
|
|
|
|
i++) {
|
|
|
|
ASSERT_OK(Put("b", "f"));
|
|
|
|
}
|
|
|
|
|
|
|
|
iter->Seek(Slice("a"));
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->b");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->d");
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterPrevWithNewerSeq) {
|
|
|
|
ASSERT_OK(Put("0", "0"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(Put("a", "b"));
|
|
|
|
ASSERT_OK(Put("c", "d"));
|
|
|
|
ASSERT_OK(Put("d", "e"));
|
|
|
|
auto iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
// Create a key that needs to be skipped for Seq too new
|
|
|
|
for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1;
|
|
|
|
i++) {
|
|
|
|
ASSERT_OK(Put("b", "f"));
|
|
|
|
}
|
|
|
|
|
|
|
|
iter->Seek(Slice("d"));
|
|
|
|
ASSERT_EQ(IterStatus(iter), "d->e");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->d");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->b");
|
|
|
|
|
|
|
|
iter->Prev();
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterPrevWithNewerSeq2) {
|
|
|
|
ASSERT_OK(Put("0", "0"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(Put("a", "b"));
|
|
|
|
ASSERT_OK(Put("c", "d"));
|
|
|
|
ASSERT_OK(Put("d", "e"));
|
|
|
|
auto iter = db_->NewIterator(ReadOptions());
|
|
|
|
iter->Seek(Slice("c"));
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->d");
|
|
|
|
|
|
|
|
// Create a key that needs to be skipped for Seq too new
|
|
|
|
for (uint64_t i = 0; i < last_options_.max_sequential_skip_in_iterations + 1;
|
|
|
|
i++) {
|
|
|
|
ASSERT_OK(Put("b", "f"));
|
|
|
|
}
|
|
|
|
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->b");
|
|
|
|
|
|
|
|
iter->Prev();
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2011-03-25 21:27:43 +01:00
|
|
|
TEST(DBTest, IterEmpty) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->Seek("foo");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
delete iter;
|
|
|
|
} while (ChangeCompactOptions());
|
2011-03-25 21:27:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterSingle) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "a", "va"));
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->Seek("a");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("b");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
} while (ChangeCompactOptions());
|
2011-03-25 21:27:43 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterMulti) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "a", "va"));
|
|
|
|
ASSERT_OK(Put(1, "b", "vb"));
|
|
|
|
ASSERT_OK(Put(1, "c", "vc"));
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Seek("a");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Seek("ax");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
2013-11-18 20:32:54 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->Seek("b");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Seek("z");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
// Switch from reverse to forward
|
|
|
|
iter->SeekToLast();
|
|
|
|
iter->Prev();
|
|
|
|
iter->Prev();
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
|
|
|
|
// Switch from forward to reverse
|
|
|
|
iter->SeekToFirst();
|
|
|
|
iter->Next();
|
|
|
|
iter->Next();
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
|
|
|
|
// Make sure iter stays at snapshot
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "va2"));
|
|
|
|
ASSERT_OK(Put(1, "a2", "va3"));
|
|
|
|
ASSERT_OK(Put(1, "b", "vb2"));
|
|
|
|
ASSERT_OK(Put(1, "c", "vc2"));
|
|
|
|
ASSERT_OK(Delete(1, "b"));
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
} while (ChangeCompactOptions());
|
2011-03-25 21:27:43 +01:00
|
|
|
}
|
|
|
|
|
2013-07-28 20:53:08 +02:00
|
|
|
// Check that we can skip over a run of user keys
|
|
|
|
// by using reseek rather than sequential scan
|
|
|
|
TEST(DBTest, IterReseek) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.max_sequential_skip_in_iterations = 3;
|
|
|
|
options.create_if_missing = true;
|
2013-10-04 06:49:15 +02:00
|
|
|
options.statistics = rocksdb::CreateDBStatistics();
|
2013-07-28 20:53:08 +02:00
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-07-28 20:53:08 +02:00
|
|
|
|
|
|
|
// insert two keys with same userkey and verify that
|
|
|
|
// reseek is not invoked. For each of these test cases,
|
|
|
|
// verify that we can find the next key "b".
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "one"));
|
|
|
|
ASSERT_OK(Put(1, "a", "two"));
|
|
|
|
ASSERT_OK(Put(1, "b", "bone"));
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->SeekToFirst();
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
|
2013-07-28 20:53:08 +02:00
|
|
|
ASSERT_EQ(IterStatus(iter), "a->two");
|
|
|
|
iter->Next();
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
|
2013-07-28 20:53:08 +02:00
|
|
|
ASSERT_EQ(IterStatus(iter), "b->bone");
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// insert a total of three keys with same userkey and verify
|
|
|
|
// that reseek is still not invoked.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "three"));
|
|
|
|
iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->three");
|
|
|
|
iter->Next();
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
|
2013-07-28 20:53:08 +02:00
|
|
|
ASSERT_EQ(IterStatus(iter), "b->bone");
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// insert a total of four keys with same userkey and verify
|
|
|
|
// that reseek is invoked.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "a", "four"));
|
|
|
|
iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->four");
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 0);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->Next();
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION), 1);
|
2013-07-28 20:53:08 +02:00
|
|
|
ASSERT_EQ(IterStatus(iter), "b->bone");
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// Testing reverse iterator
|
|
|
|
// At this point, we have three versions of "a" and one version of "b".
|
|
|
|
// The reseek statistics is already at 1.
|
2014-01-17 21:46:06 +01:00
|
|
|
int num_reseeks =
|
|
|
|
(int)TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION);
|
2013-07-28 20:53:08 +02:00
|
|
|
|
|
|
|
// Insert another version of b and assert that reseek is not invoked
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "b", "btwo"));
|
|
|
|
iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->btwo");
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
|
|
|
|
num_reseeks);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->Prev();
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
|
|
|
|
num_reseeks + 1);
|
2013-07-28 20:53:08 +02:00
|
|
|
ASSERT_EQ(IterStatus(iter), "a->four");
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// insert two more versions of b. This makes a total of 4 versions
|
|
|
|
// of b and 4 versions of a.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "b", "bthree"));
|
|
|
|
ASSERT_OK(Put(1, "b", "bfour"));
|
|
|
|
iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->bfour");
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
|
|
|
|
num_reseeks + 2);
|
2013-07-28 20:53:08 +02:00
|
|
|
iter->Prev();
|
|
|
|
|
|
|
|
// the previous Prev call should have invoked reseek
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, NUMBER_OF_RESEEKS_IN_ITERATION),
|
|
|
|
num_reseeks + 3);
|
2013-07-28 20:53:08 +02:00
|
|
|
ASSERT_EQ(IterStatus(iter), "a->four");
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2011-03-25 21:27:43 +01:00
|
|
|
TEST(DBTest, IterSmallAndLargeMix) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "a", "va"));
|
|
|
|
ASSERT_OK(Put(1, "b", std::string(100000, 'b')));
|
|
|
|
ASSERT_OK(Put(1, "c", "vc"));
|
|
|
|
ASSERT_OK(Put(1, "d", std::string(100000, 'd')));
|
|
|
|
ASSERT_OK(Put(1, "e", std::string(100000, 'e')));
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
} while (ChangeCompactOptions());
|
2011-03-25 21:27:43 +01:00
|
|
|
}
|
|
|
|
|
2011-08-16 03:21:01 +02:00
|
|
|
TEST(DBTest, IterMultiWithDelete) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2014-04-25 21:21:34 +02:00
|
|
|
ASSERT_OK(Put(1, "ka", "va"));
|
|
|
|
ASSERT_OK(Put(1, "kb", "vb"));
|
|
|
|
ASSERT_OK(Put(1, "kc", "vc"));
|
|
|
|
ASSERT_OK(Delete(1, "kb"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, "kb"));
|
2014-02-07 23:47:16 +01:00
|
|
|
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2014-04-25 21:21:34 +02:00
|
|
|
iter->Seek("kc");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "kc->vc");
|
2013-03-21 23:59:47 +01:00
|
|
|
if (!CurrentOptions().merge_operator) {
|
|
|
|
// TODO: merge operator does not support backward iteration yet
|
2014-04-25 21:21:34 +02:00
|
|
|
if (kPlainTableAllBytesPrefix != option_config_&&
|
|
|
|
kBlockBasedTableWithWholeKeyHashIndex != option_config_ &&
|
|
|
|
kHashLinkList != option_config_) {
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "ka->va");
|
|
|
|
}
|
2013-03-21 23:59:47 +01:00
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
delete iter;
|
|
|
|
} while (ChangeOptions());
|
2011-08-16 03:21:01 +02:00
|
|
|
}
|
|
|
|
|
2013-10-18 03:33:18 +02:00
|
|
|
TEST(DBTest, IterPrevMaxSkip) {
|
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2013-10-18 03:33:18 +02:00
|
|
|
for (int i = 0; i < 2; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key1", "v1"));
|
|
|
|
ASSERT_OK(Put(1, "key2", "v2"));
|
|
|
|
ASSERT_OK(Put(1, "key3", "v3"));
|
|
|
|
ASSERT_OK(Put(1, "key4", "v4"));
|
|
|
|
ASSERT_OK(Put(1, "key5", "v5"));
|
2013-10-18 03:33:18 +02:00
|
|
|
}
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
VerifyIterLast("key5->v5", 1);
|
2013-10-18 03:33:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "key5"));
|
|
|
|
VerifyIterLast("key4->v4", 1);
|
2013-10-18 03:33:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "key4"));
|
|
|
|
VerifyIterLast("key3->v3", 1);
|
2013-10-18 03:33:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "key3"));
|
|
|
|
VerifyIterLast("key2->v2", 1);
|
2013-10-18 03:33:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "key2"));
|
|
|
|
VerifyIterLast("key1->v1", 1);
|
2013-10-18 03:33:18 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "key1"));
|
|
|
|
VerifyIterLast("(invalid)", 1);
|
2014-04-25 21:21:34 +02:00
|
|
|
} while (ChangeOptions(kSkipMergePut | kSkipNoSeekToLast));
|
2013-10-18 03:33:18 +02:00
|
|
|
}
|
|
|
|
|
2013-09-28 20:39:08 +02:00
|
|
|
TEST(DBTest, IterWithSnapshot) {
|
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "key1", "val1"));
|
|
|
|
ASSERT_OK(Put(1, "key2", "val2"));
|
|
|
|
ASSERT_OK(Put(1, "key3", "val3"));
|
|
|
|
ASSERT_OK(Put(1, "key4", "val4"));
|
|
|
|
ASSERT_OK(Put(1, "key5", "val5"));
|
2013-09-28 20:39:08 +02:00
|
|
|
|
|
|
|
const Snapshot *snapshot = db_->GetSnapshot();
|
|
|
|
ReadOptions options;
|
|
|
|
options.snapshot = snapshot;
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = db_->NewIterator(options, handles_[1]);
|
2013-09-28 20:39:08 +02:00
|
|
|
|
|
|
|
// Put more values after the snapshot
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key100", "val100"));
|
|
|
|
ASSERT_OK(Put(1, "key101", "val101"));
|
2013-09-28 20:39:08 +02:00
|
|
|
|
|
|
|
iter->Seek("key5");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "key5->val5");
|
|
|
|
if (!CurrentOptions().merge_operator) {
|
|
|
|
// TODO: merge operator does not support backward iteration yet
|
2014-04-25 21:21:34 +02:00
|
|
|
if (kPlainTableAllBytesPrefix != option_config_&&
|
|
|
|
kBlockBasedTableWithWholeKeyHashIndex != option_config_ &&
|
|
|
|
kHashLinkList != option_config_) {
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "key4->val4");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "key3->val3");
|
2013-09-28 20:39:08 +02:00
|
|
|
|
2014-04-25 21:21:34 +02:00
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "key4->val4");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "key5->val5");
|
|
|
|
}
|
2013-09-28 20:39:08 +02:00
|
|
|
iter->Next();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
}
|
|
|
|
db_->ReleaseSnapshot(snapshot);
|
|
|
|
delete iter;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// skip as HashCuckooRep does not support snapshot
|
|
|
|
} while (ChangeOptions(kSkipHashCuckoo));
|
2013-09-28 20:39:08 +02:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, Recover) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_OK(Put(1, "baz", "v5"));
|
|
|
|
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v5", Get(1, "baz"));
|
|
|
|
ASSERT_OK(Put(1, "bar", "v2"));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v3"));
|
|
|
|
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("v3", Get(1, "foo"));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v4"));
|
|
|
|
ASSERT_EQ("v4", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v2", Get(1, "bar"));
|
|
|
|
ASSERT_EQ("v5", Get(1, "baz"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2014-02-12 19:43:27 +01:00
|
|
|
TEST(DBTest, RecoverWithTableHandle) {
|
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2014-02-12 19:43:27 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.write_buffer_size = 100;
|
|
|
|
options.disable_auto_compactions = true;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-12 19:43:27 +01:00
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2014-02-12 19:43:27 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_OK(Put(1, "bar", "v2"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v3"));
|
|
|
|
ASSERT_OK(Put(1, "bar", "v4"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_OK(Put(1, "big", std::string(100, 'a')));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2014-02-12 19:43:27 +01:00
|
|
|
|
|
|
|
std::vector<std::vector<FileMetaData>> files;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_GetFilesMetaData(handles_[1], &files);
|
2014-02-12 19:43:27 +01:00
|
|
|
int total_files = 0;
|
|
|
|
for (const auto& level : files) {
|
|
|
|
total_files += level.size();
|
|
|
|
}
|
|
|
|
ASSERT_EQ(total_files, 3);
|
|
|
|
for (const auto& level : files) {
|
|
|
|
for (const auto& file : level) {
|
|
|
|
if (kInfiniteMaxOpenFiles == option_config_) {
|
|
|
|
ASSERT_TRUE(file.table_reader_handle != nullptr);
|
|
|
|
} else {
|
|
|
|
ASSERT_TRUE(file.table_reader_handle == nullptr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} while (ChangeOptions());
|
|
|
|
}
|
|
|
|
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
TEST(DBTest, IgnoreRecoveredLog) {
|
|
|
|
std::string backup_logs = dbname_ + "/backup_logs";
|
|
|
|
|
|
|
|
// delete old files in backup_logs directory
|
|
|
|
env_->CreateDirIfMissing(backup_logs);
|
|
|
|
std::vector<std::string> old_files;
|
|
|
|
env_->GetChildren(backup_logs, &old_files);
|
|
|
|
for (auto& file : old_files) {
|
|
|
|
if (file != "." && file != "..") {
|
|
|
|
env_->DeleteFile(backup_logs + "/" + file);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
|
|
|
|
options.wal_dir = dbname_ + "/logs";
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
// fill up the DB
|
|
|
|
std::string one, two;
|
|
|
|
PutFixed64(&one, 1);
|
|
|
|
PutFixed64(&two, 2);
|
|
|
|
ASSERT_OK(db_->Merge(WriteOptions(), Slice("foo"), Slice(one)));
|
|
|
|
ASSERT_OK(db_->Merge(WriteOptions(), Slice("foo"), Slice(one)));
|
|
|
|
ASSERT_OK(db_->Merge(WriteOptions(), Slice("bar"), Slice(one)));
|
|
|
|
|
|
|
|
// copy the logs to backup
|
|
|
|
std::vector<std::string> logs;
|
|
|
|
env_->GetChildren(options.wal_dir, &logs);
|
|
|
|
for (auto& log : logs) {
|
|
|
|
if (log != ".." && log != ".") {
|
|
|
|
CopyFile(options.wal_dir + "/" + log, backup_logs + "/" + log);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// recover the DB
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ(two, Get("foo"));
|
|
|
|
ASSERT_EQ(one, Get("bar"));
|
|
|
|
Close();
|
|
|
|
|
|
|
|
// copy the logs from backup back to wal dir
|
|
|
|
for (auto& log : logs) {
|
|
|
|
if (log != ".." && log != ".") {
|
|
|
|
CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// this should ignore the log files, recovery should not happen again
|
|
|
|
// if the recovery happens, the same merge operator would be called twice,
|
|
|
|
// leading to incorrect results
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ(two, Get("foo"));
|
|
|
|
ASSERT_EQ(one, Get("bar"));
|
|
|
|
Close();
|
|
|
|
Destroy(&options);
|
2014-04-15 18:06:13 +02:00
|
|
|
Reopen(&options);
|
|
|
|
Close();
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
|
|
|
|
// copy the logs from backup back to wal dir
|
|
|
|
env_->CreateDirIfMissing(options.wal_dir);
|
|
|
|
for (auto& log : logs) {
|
|
|
|
if (log != ".." && log != ".") {
|
|
|
|
CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// assert that we successfully recovered only from logs, even though we
|
|
|
|
// destroyed the DB
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ(two, Get("foo"));
|
|
|
|
ASSERT_EQ(one, Get("bar"));
|
2014-04-15 18:06:13 +02:00
|
|
|
|
|
|
|
// Recovery will fail if DB directory doesn't exist.
|
|
|
|
Destroy(&options);
|
|
|
|
// copy the logs from backup back to wal dir
|
|
|
|
env_->CreateDirIfMissing(options.wal_dir);
|
|
|
|
for (auto& log : logs) {
|
|
|
|
if (log != ".." && log != ".") {
|
|
|
|
CopyFile(backup_logs + "/" + log, options.wal_dir + "/" + log);
|
|
|
|
// we won't be needing this file no more
|
|
|
|
env_->DeleteFile(backup_logs + "/" + log);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Status s = TryReopen(&options);
|
|
|
|
ASSERT_TRUE(!s.ok());
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
} while (ChangeOptions(kSkipHashCuckoo));
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
}
|
|
|
|
|
2012-08-18 01:06:05 +02:00
|
|
|
TEST(DBTest, RollLog) {
|
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_OK(Put(1, "baz", "v5"));
|
2012-08-18 01:06:05 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2012-08-18 01:06:05 +02:00
|
|
|
for (int i = 0; i < 10; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2012-08-18 01:06:05 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "v4"));
|
2012-08-18 01:06:05 +02:00
|
|
|
for (int i = 0; i < 10; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2012-08-18 01:06:05 +02:00
|
|
|
}
|
|
|
|
} while (ChangeOptions());
|
|
|
|
}
|
|
|
|
|
2012-07-05 22:39:28 +02:00
|
|
|
TEST(DBTest, WAL) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-03-12 19:50:10 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2013-08-08 00:20:41 +02:00
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
2014-03-12 19:50:10 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v1", Get(1, "bar"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
writeOpt.disableWAL = false;
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
|
2013-08-08 00:20:41 +02:00
|
|
|
writeOpt.disableWAL = true;
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
2014-03-12 19:50:10 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2013-08-08 00:20:41 +02:00
|
|
|
// Both value's should be present.
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_EQ("v2", Get(1, "bar"));
|
|
|
|
ASSERT_EQ("v2", Get(1, "foo"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
writeOpt.disableWAL = true;
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
|
2013-08-08 00:20:41 +02:00
|
|
|
writeOpt.disableWAL = false;
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
2014-03-12 19:50:10 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2013-08-08 00:20:41 +02:00
|
|
|
// again both values should be present.
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_EQ("v3", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v3", Get(1, "bar"));
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2012-07-05 22:39:28 +02:00
|
|
|
}
|
|
|
|
|
2012-08-18 09:26:50 +02:00
|
|
|
TEST(DBTest, CheckLock) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
DB* localdb;
|
|
|
|
Options options = CurrentOptions();
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(TryReopen(&options));
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
// second open should fail
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2012-08-18 09:26:50 +02:00
|
|
|
}
|
|
|
|
|
2013-09-05 02:24:35 +02:00
|
|
|
TEST(DBTest, FlushMultipleMemtable) {
|
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
options.max_write_buffer_number = 4;
|
|
|
|
options.min_write_buffer_number_to_merge = 3;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
|
|
|
|
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v1", Get(1, "bar"));
|
|
|
|
ASSERT_OK(Flush(1));
|
2013-09-05 02:24:35 +02:00
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
2013-10-05 00:17:54 +02:00
|
|
|
TEST(DBTest, NumImmutableMemTable) {
|
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
options.max_write_buffer_number = 4;
|
|
|
|
options.min_write_buffer_number_to_merge = 3;
|
|
|
|
options.write_buffer_size = 1000000;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-10-05 00:17:54 +02:00
|
|
|
|
2014-03-13 00:40:14 +01:00
|
|
|
std::string big_value(1000000 * 2, 'x');
|
2013-10-05 00:17:54 +02:00
|
|
|
std::string num;
|
2013-11-18 20:32:54 +01:00
|
|
|
SetPerfLevel(kEnableTime);;
|
2013-10-05 00:17:54 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k1", big_value));
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
|
|
|
|
"rocksdb.num-immutable-mem-table", &num));
|
2013-10-05 00:17:54 +02:00
|
|
|
ASSERT_EQ(num, "0");
|
2014-04-23 02:17:33 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
2013-11-18 20:32:54 +01:00
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "k1");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_EQ(1, (int) perf_context.get_from_memtable_count);
|
2013-10-05 00:17:54 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
|
|
|
|
"rocksdb.num-immutable-mem-table", &num));
|
2013-10-05 00:17:54 +02:00
|
|
|
ASSERT_EQ(num, "1");
|
2014-04-23 02:17:33 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
|
|
|
|
2013-11-18 20:32:54 +01:00
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "k1");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_EQ(2, (int) perf_context.get_from_memtable_count);
|
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "k2");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_EQ(1, (int) perf_context.get_from_memtable_count);
|
2013-10-05 00:17:54 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value));
|
2014-03-31 21:44:54 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.cur-size-active-mem-table", &num));
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
|
|
|
|
"rocksdb.num-immutable-mem-table", &num));
|
2013-10-05 00:17:54 +02:00
|
|
|
ASSERT_EQ(num, "2");
|
2014-04-23 02:17:33 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-active-mem-table", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
|
|
|
|
ASSERT_EQ(num, "2");
|
2013-11-18 20:32:54 +01:00
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "k2");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_EQ(2, (int) perf_context.get_from_memtable_count);
|
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "k3");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_EQ(1, (int) perf_context.get_from_memtable_count);
|
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "k1");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_EQ(3, (int) perf_context.get_from_memtable_count);
|
2013-10-05 00:17:54 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
|
|
|
|
"rocksdb.num-immutable-mem-table", &num));
|
2013-10-05 00:17:54 +02:00
|
|
|
ASSERT_EQ(num, "0");
|
2014-03-31 21:44:54 +02:00
|
|
|
ASSERT_TRUE(dbfull()->GetProperty(
|
|
|
|
handles_[1], "rocksdb.cur-size-active-mem-table", &num));
|
2014-05-09 20:01:54 +02:00
|
|
|
// "200" is the size of the metadata of an empty skiplist, this would
|
2014-03-27 19:59:37 +01:00
|
|
|
// break if we change the default skiplist implementation
|
2014-05-09 20:01:54 +02:00
|
|
|
ASSERT_EQ(num, "200");
|
2013-11-18 20:32:54 +01:00
|
|
|
SetPerfLevel(kDisable);
|
2013-10-05 00:17:54 +02:00
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
2014-03-18 20:37:42 +01:00
|
|
|
class SleepingBackgroundTask {
|
|
|
|
public:
|
2014-04-08 23:57:00 +02:00
|
|
|
SleepingBackgroundTask()
|
|
|
|
: bg_cv_(&mutex_), should_sleep_(true), done_with_sleep_(false) {}
|
2014-03-18 20:37:42 +01:00
|
|
|
void DoSleep() {
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
while (should_sleep_) {
|
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
2014-04-08 23:57:00 +02:00
|
|
|
done_with_sleep_ = true;
|
|
|
|
bg_cv_.SignalAll();
|
2014-03-18 20:37:42 +01:00
|
|
|
}
|
|
|
|
void WakeUp() {
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
should_sleep_ = false;
|
|
|
|
bg_cv_.SignalAll();
|
|
|
|
}
|
2014-04-08 23:57:00 +02:00
|
|
|
void WaitUntilDone() {
|
|
|
|
MutexLock l(&mutex_);
|
|
|
|
while (!done_with_sleep_) {
|
|
|
|
bg_cv_.Wait();
|
|
|
|
}
|
|
|
|
}
|
2014-03-18 20:37:42 +01:00
|
|
|
|
|
|
|
static void DoSleepTask(void* arg) {
|
|
|
|
reinterpret_cast<SleepingBackgroundTask*>(arg)->DoSleep();
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
port::Mutex mutex_;
|
|
|
|
port::CondVar bg_cv_; // Signalled when background work finishes
|
|
|
|
bool should_sleep_;
|
2014-04-08 23:57:00 +02:00
|
|
|
bool done_with_sleep_;
|
2014-03-18 20:37:42 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
TEST(DBTest, GetProperty) {
|
|
|
|
// Set sizes to both background thread pool to be 1 and block them.
|
|
|
|
env_->SetBackgroundThreads(1, Env::HIGH);
|
|
|
|
env_->SetBackgroundThreads(1, Env::LOW);
|
2014-03-19 23:40:12 +01:00
|
|
|
SleepingBackgroundTask sleeping_task_low;
|
2014-03-18 20:37:42 +01:00
|
|
|
env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
|
|
|
|
Env::Priority::LOW);
|
2014-03-19 23:40:12 +01:00
|
|
|
SleepingBackgroundTask sleeping_task_high;
|
2014-03-18 20:37:42 +01:00
|
|
|
env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_high,
|
|
|
|
Env::Priority::HIGH);
|
|
|
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.level0_file_num_compaction_trigger = 1;
|
|
|
|
options.compaction_options_universal.size_ratio = 50;
|
|
|
|
options.max_background_compactions = 1;
|
|
|
|
options.max_background_flushes = 1;
|
|
|
|
options.max_write_buffer_number = 10;
|
|
|
|
options.min_write_buffer_number_to_merge = 1;
|
|
|
|
options.write_buffer_size = 1000000;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
std::string big_value(1000000 * 2, 'x');
|
|
|
|
std::string num;
|
|
|
|
SetPerfLevel(kEnableTime);
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "k1", big_value));
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
|
|
|
|
ASSERT_EQ(num, "0");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
|
|
|
|
ASSERT_EQ(num, "0");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
|
|
|
|
ASSERT_EQ(num, "0");
|
|
|
|
perf_context.Reset();
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
|
|
|
|
ASSERT_EQ(num, "2");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
|
|
|
|
ASSERT_EQ(num, "0");
|
|
|
|
|
|
|
|
sleeping_task_high.WakeUp();
|
2014-04-08 23:57:00 +02:00
|
|
|
sleeping_task_high.WaitUntilDone();
|
2014-03-18 20:37:42 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "k4", big_value));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "k5", big_value));
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.mem-table-flush-pending", &num));
|
|
|
|
ASSERT_EQ(num, "0");
|
|
|
|
ASSERT_TRUE(dbfull()->GetProperty("rocksdb.compaction-pending", &num));
|
|
|
|
ASSERT_EQ(num, "1");
|
|
|
|
sleeping_task_low.WakeUp();
|
2014-04-08 23:57:00 +02:00
|
|
|
sleeping_task_low.WaitUntilDone();
|
2014-03-18 20:37:42 +01:00
|
|
|
}
|
|
|
|
|
2012-07-06 20:42:09 +02:00
|
|
|
TEST(DBTest, FLUSH) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2013-08-08 00:20:41 +02:00
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
2013-11-18 20:32:54 +01:00
|
|
|
SetPerfLevel(kEnableTime);;
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
|
2013-08-08 00:20:41 +02:00
|
|
|
// this will now also flush the last 2 writes
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
|
2012-07-06 20:42:09 +02:00
|
|
|
|
2013-11-18 20:32:54 +01:00
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
Get(1, "foo");
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_TRUE((int) perf_context.get_from_output_files_time > 0);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
2014-03-12 19:50:10 +01:00
|
|
|
ASSERT_EQ("v1", Get(1, "bar"));
|
2012-07-06 20:42:09 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
writeOpt.disableWAL = true;
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
|
|
|
|
ASSERT_OK(Flush(1));
|
2012-07-06 20:42:09 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("v2", Get(1, "bar"));
|
2013-11-18 20:32:54 +01:00
|
|
|
perf_context.Reset();
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("v2", Get(1, "foo"));
|
2013-11-18 20:32:54 +01:00
|
|
|
ASSERT_TRUE((int) perf_context.get_from_output_files_time > 0);
|
2012-07-06 20:42:09 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
writeOpt.disableWAL = false;
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
|
|
|
|
ASSERT_OK(Flush(1));
|
2012-07-06 20:42:09 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
2013-08-08 00:20:41 +02:00
|
|
|
// 'foo' should be there because its put
|
|
|
|
// has WAL enabled.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("v3", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v3", Get(1, "bar"));
|
2013-11-18 20:32:54 +01:00
|
|
|
|
|
|
|
SetPerfLevel(kDisable);
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2012-07-06 20:42:09 +02:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, RecoveryWithEmptyLog) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v1"));
|
|
|
|
ASSERT_OK(Put(1, "foo", "v2"));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v3"));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("v3", Get(1, "foo"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Check that writes done during a memtable compaction are recovered
|
|
|
|
// if the database is shutdown during the memtable compaction.
|
|
|
|
TEST(DBTest, RecoverDuringMemtableCompaction) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2012-04-17 17:36:46 +02:00
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 1000000;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Trigger a long memtable compaction and reopen the database during it
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "v1")); // Goes to 1st log file
|
|
|
|
ASSERT_OK(Put(1, "big1", std::string(10000000, 'x'))); // Fills memtable
|
|
|
|
ASSERT_OK(Put(1, "big2", std::string(1000, 'y'))); // Triggers compaction
|
|
|
|
ASSERT_OK(Put(1, "bar", "v2")); // Goes to new log file
|
|
|
|
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
ASSERT_EQ("v1", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("v2", Get(1, "bar"));
|
|
|
|
ASSERT_EQ(std::string(10000000, 'x'), Get(1, "big1"));
|
|
|
|
ASSERT_EQ(std::string(1000, 'y'), Get(1, "big2"));
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, MinorCompactionsHappen) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.write_buffer_size = 10000;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
const int N = 500;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
int starting_num_tables = TotalTableFiles(1);
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), Key(i) + std::string(1000, 'v')));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
int ending_num_tables = TotalTableFiles(1);
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_GT(ending_num_tables, starting_num_tables);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(1, Key(i)));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(1, Key(i)));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
} while (ChangeCompactOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-01-11 02:18:50 +01:00
|
|
|
TEST(DBTest, ManifestRollOver) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.max_manifest_file_size = 10 ; // 10 bytes
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-08 00:20:41 +02:00
|
|
|
{
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
|
|
|
|
ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
|
|
|
|
ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
|
|
|
|
uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
|
|
|
|
ASSERT_OK(Flush(1)); // This should trigger LogAndApply.
|
|
|
|
uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
|
2013-10-18 23:50:54 +02:00
|
|
|
ASSERT_GT(manifest_after_flush, manifest_before_flush);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
|
2013-08-08 00:20:41 +02:00
|
|
|
// check if a new manifest file got inserted or not.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
|
|
|
|
ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
|
|
|
|
ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
} while (ChangeCompactOptions());
|
2013-01-11 02:18:50 +01:00
|
|
|
}
|
|
|
|
|
2013-10-18 23:50:54 +02:00
|
|
|
TEST(DBTest, IdentityAcrossRestarts) {
|
|
|
|
do {
|
2013-12-03 15:39:07 +01:00
|
|
|
std::string id1;
|
|
|
|
ASSERT_OK(db_->GetDbIdentity(id1));
|
2013-10-18 23:50:54 +02:00
|
|
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
Reopen(&options);
|
2013-12-03 15:39:07 +01:00
|
|
|
std::string id2;
|
|
|
|
ASSERT_OK(db_->GetDbIdentity(id2));
|
2013-10-18 23:50:54 +02:00
|
|
|
// id1 should match id2 because identity was not regenerated
|
2013-12-03 15:39:07 +01:00
|
|
|
ASSERT_EQ(id1.compare(id2), 0);
|
2013-10-18 23:50:54 +02:00
|
|
|
|
2013-12-03 15:39:07 +01:00
|
|
|
std::string idfilename = IdentityFileName(dbname_);
|
2013-10-18 23:50:54 +02:00
|
|
|
ASSERT_OK(env_->DeleteFile(idfilename));
|
|
|
|
Reopen(&options);
|
2013-12-03 15:39:07 +01:00
|
|
|
std::string id3;
|
|
|
|
ASSERT_OK(db_->GetDbIdentity(id3));
|
|
|
|
// id1 should NOT match id3 because identity was regenerated
|
|
|
|
ASSERT_NE(id1.compare(id3), 0);
|
2013-10-18 23:50:54 +02:00
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, RecoverWithLargeLog) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
{
|
|
|
|
Options options = CurrentOptions();
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
|
|
|
ASSERT_OK(Put(1, "big1", std::string(200000, '1')));
|
|
|
|
ASSERT_OK(Put(1, "big2", std::string(200000, '2')));
|
|
|
|
ASSERT_OK(Put(1, "small3", std::string(10, '3')));
|
|
|
|
ASSERT_OK(Put(1, "small4", std::string(10, '4')));
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure that if we re-open with a small write buffer size that
|
|
|
|
// we flush table files in the middle of a large log file.
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.write_buffer_size = 100000;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 3);
|
|
|
|
ASSERT_EQ(std::string(200000, '1'), Get(1, "big1"));
|
|
|
|
ASSERT_EQ(std::string(200000, '2'), Get(1, "big2"));
|
|
|
|
ASSERT_EQ(std::string(10, '3'), Get(1, "small3"));
|
|
|
|
ASSERT_EQ(std::string(10, '4'), Get(1, "small4"));
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(0, 1), 1);
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionsGenerateMultipleFiles) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2011-03-18 23:37:00 +01:00
|
|
|
options.write_buffer_size = 100000000; // Large write buffer
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Write 8MB (80 values, each 100K)
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
2011-03-18 23:37:00 +01:00
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 80; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 100000));
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), values[i]));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Reopening moves updates to level-0
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(1, 1), 1);
|
2011-03-18 23:37:00 +01:00
|
|
|
for (int i = 0; i < 80; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Get(1, Key(i)), values[i]);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-06-23 04:30:03 +02:00
|
|
|
TEST(DBTest, CompactionTrigger) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2012-06-23 04:30:03 +02:00
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2012-06-23 04:30:03 +02:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
2012-11-27 06:16:21 +01:00
|
|
|
num++) {
|
2012-06-23 04:30:03 +02:00
|
|
|
std::vector<std::string> values;
|
|
|
|
// Write 120KB (12 values, each 10K)
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), values[i]));
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 1);
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
//generate one more file in level-0, and should trigger level-0 compaction
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), values[i]));
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 1);
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
2014-06-25 00:37:06 +02:00
|
|
|
namespace {
|
|
|
|
static const int kCDTValueSize = 1000;
|
|
|
|
static const int kCDTKeysPerBuffer = 4;
|
|
|
|
static const int kCDTNumLevels = 8;
|
|
|
|
Options DeletionTriggerOptions() {
|
|
|
|
Options options;
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
options.write_buffer_size = kCDTKeysPerBuffer * (kCDTValueSize + 24);
|
|
|
|
options.min_write_buffer_number_to_merge = 1;
|
|
|
|
options.num_levels = kCDTNumLevels;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
options.level0_file_num_compaction_trigger = 1;
|
|
|
|
options.target_file_size_base = options.write_buffer_size * 2;
|
|
|
|
options.target_file_size_multiplier = 2;
|
|
|
|
options.max_bytes_for_level_base =
|
|
|
|
options.target_file_size_base * options.target_file_size_multiplier;
|
|
|
|
options.max_bytes_for_level_multiplier = 2;
|
|
|
|
options.disable_auto_compactions = false;
|
|
|
|
return options;
|
|
|
|
}
|
|
|
|
} // anonymous namespace
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionDeletionTrigger) {
|
|
|
|
Options options = DeletionTriggerOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
|
|
|
|
for (int tid = 0; tid < 2; ++tid) {
|
|
|
|
uint64_t db_size[2];
|
|
|
|
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
const int kTestSize = kCDTKeysPerBuffer * 512;
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int k = 0; k < kTestSize; ++k) {
|
|
|
|
values.push_back(RandomString(&rnd, kCDTValueSize));
|
|
|
|
ASSERT_OK(Put(Key(k), values[k]));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
db_size[0] = Size(Key(0), Key(kTestSize - 1));
|
|
|
|
|
|
|
|
for (int k = 0; k < kTestSize; ++k) {
|
|
|
|
ASSERT_OK(Delete(Key(k)));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
db_size[1] = Size(Key(0), Key(kTestSize - 1));
|
|
|
|
|
|
|
|
// must have much smaller db size.
|
|
|
|
ASSERT_GT(db_size[0] / 3, db_size[1]);
|
|
|
|
|
|
|
|
// repeat the test with universal compaction
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.num_levels = 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionDeletionTriggerReopen) {
|
|
|
|
for (int tid = 0; tid < 2; ++tid) {
|
|
|
|
uint64_t db_size[3];
|
|
|
|
Options options = DeletionTriggerOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// round 1 --- insert key/value pairs.
|
|
|
|
const int kTestSize = kCDTKeysPerBuffer * 512;
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int k = 0; k < kTestSize; ++k) {
|
|
|
|
values.push_back(RandomString(&rnd, kCDTValueSize));
|
|
|
|
ASSERT_OK(Put(Key(k), values[k]));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
db_size[0] = Size(Key(0), Key(kTestSize - 1));
|
|
|
|
Close();
|
|
|
|
|
|
|
|
// round 2 --- disable auto-compactions and issue deletions.
|
|
|
|
options.create_if_missing = false;
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
for (int k = 0; k < kTestSize; ++k) {
|
|
|
|
ASSERT_OK(Delete(Key(k)));
|
|
|
|
}
|
|
|
|
db_size[1] = Size(Key(0), Key(kTestSize - 1));
|
|
|
|
Close();
|
|
|
|
// as auto_compaction is off, we shouldn't see too much reduce
|
|
|
|
// in db size.
|
|
|
|
ASSERT_LT(db_size[0] / 3, db_size[1]);
|
|
|
|
|
|
|
|
// round 3 --- reopen db with auto_compaction on and see if
|
|
|
|
// deletion compensation still work.
|
|
|
|
options.disable_auto_compactions = false;
|
|
|
|
Reopen(&options);
|
|
|
|
// insert relatively small amount of data to trigger auto compaction.
|
|
|
|
for (int k = 0; k < kTestSize / 10; ++k) {
|
|
|
|
ASSERT_OK(Put(Key(k), values[k]));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
db_size[2] = Size(Key(0), Key(kTestSize - 1));
|
|
|
|
// this time we're expecting significant drop in size.
|
|
|
|
ASSERT_GT(db_size[0] / 3, db_size[2]);
|
|
|
|
|
|
|
|
// repeat the test with universal compaction
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.num_levels = 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-20 00:20:29 +01:00
|
|
|
// This is a static filter used for filtering
|
|
|
|
// kvs during the compaction process.
|
|
|
|
static int cfilter_count;
|
|
|
|
static std::string NEW_VALUE = "NewValue";
|
|
|
|
|
|
|
|
class KeepFilter : public CompactionFilter {
|
|
|
|
public:
|
|
|
|
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
|
|
|
std::string* new_value, bool* value_changed) const
|
|
|
|
override {
|
|
|
|
cfilter_count++;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override { return "KeepFilter"; }
|
|
|
|
};
|
|
|
|
|
|
|
|
class DeleteFilter : public CompactionFilter {
|
|
|
|
public:
|
|
|
|
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
|
|
|
std::string* new_value, bool* value_changed) const
|
|
|
|
override {
|
|
|
|
cfilter_count++;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override { return "DeleteFilter"; }
|
|
|
|
};
|
|
|
|
|
|
|
|
class ChangeFilter : public CompactionFilter {
|
|
|
|
public:
|
|
|
|
explicit ChangeFilter() {}
|
|
|
|
|
|
|
|
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
|
|
|
std::string* new_value, bool* value_changed) const
|
|
|
|
override {
|
|
|
|
assert(new_value != nullptr);
|
|
|
|
*new_value = NEW_VALUE;
|
|
|
|
*value_changed = true;
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override { return "ChangeFilter"; }
|
|
|
|
};
|
|
|
|
|
|
|
|
class KeepFilterFactory : public CompactionFilterFactory {
|
|
|
|
public:
|
|
|
|
explicit KeepFilterFactory(bool check_context = false)
|
|
|
|
: check_context_(check_context) {}
|
|
|
|
|
|
|
|
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
2014-04-02 23:48:53 +02:00
|
|
|
const CompactionFilter::Context& context) override {
|
2014-03-20 00:20:29 +01:00
|
|
|
if (check_context_) {
|
|
|
|
ASSERT_EQ(expect_full_compaction_.load(), context.is_full_compaction);
|
|
|
|
ASSERT_EQ(expect_manual_compaction_.load(), context.is_manual_compaction);
|
|
|
|
}
|
|
|
|
return std::unique_ptr<CompactionFilter>(new KeepFilter());
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override { return "KeepFilterFactory"; }
|
|
|
|
bool check_context_;
|
|
|
|
std::atomic_bool expect_full_compaction_;
|
|
|
|
std::atomic_bool expect_manual_compaction_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class DeleteFilterFactory : public CompactionFilterFactory {
|
|
|
|
public:
|
|
|
|
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
2014-04-02 23:48:53 +02:00
|
|
|
const CompactionFilter::Context& context) override {
|
2014-03-20 00:20:29 +01:00
|
|
|
if (context.is_manual_compaction) {
|
|
|
|
return std::unique_ptr<CompactionFilter>(new DeleteFilter());
|
|
|
|
} else {
|
|
|
|
return std::unique_ptr<CompactionFilter>(nullptr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override { return "DeleteFilterFactory"; }
|
|
|
|
};
|
|
|
|
|
|
|
|
class ChangeFilterFactory : public CompactionFilterFactory {
|
|
|
|
public:
|
|
|
|
explicit ChangeFilterFactory() {}
|
|
|
|
|
|
|
|
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
2014-04-02 23:48:53 +02:00
|
|
|
const CompactionFilter::Context& context) override {
|
2014-03-20 00:20:29 +01:00
|
|
|
return std::unique_ptr<CompactionFilter>(new ChangeFilter());
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override { return "ChangeFilterFactory"; }
|
|
|
|
};
|
|
|
|
|
2014-03-13 00:40:14 +01:00
|
|
|
// TODO(kailiu) The tests on UniversalCompaction has some issues:
|
|
|
|
// 1. A lot of magic numbers ("11" or "12").
|
|
|
|
// 2. Made assumption on the memtable flush conidtions, which may change from
|
|
|
|
// time to time.
|
2013-08-08 00:20:41 +02:00
|
|
|
TEST(DBTest, UniversalCompactionTrigger) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
2013-09-17 22:56:30 +02:00
|
|
|
// trigger compaction if there are >= 4 files
|
|
|
|
options.level0_file_num_compaction_trigger = 4;
|
2014-03-20 00:20:29 +01:00
|
|
|
KeepFilterFactory* filter = new KeepFilterFactory(true);
|
|
|
|
filter->expect_manual_compaction_.store(false);
|
|
|
|
options.compaction_filter_factory.reset(filter);
|
|
|
|
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
int key_idx = 0;
|
|
|
|
|
2014-03-20 00:20:29 +01:00
|
|
|
filter->expect_full_compaction_.store(true);
|
2013-08-08 00:20:41 +02:00
|
|
|
// Stage 1:
|
|
|
|
// Generate a set of files at level 0, but don't trigger level-0
|
|
|
|
// compaction.
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
2013-08-08 00:20:41 +02:00
|
|
|
num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < 12; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 1);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Generate one more file at level-0, which should trigger level-0
|
|
|
|
// compaction.
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Suppose each file flushed from mem table has size 1. Now we compact
|
|
|
|
// (level0_file_num_compaction_trigger+1)=4 files and should have a big
|
|
|
|
// file of size 4.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 1; i < options.num_levels ; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Stage 2:
|
|
|
|
// Now we have one file at level 0, with size 4. We also have some data in
|
|
|
|
// mem table. Let's continue generating new files at level 0, but don't
|
|
|
|
// trigger level-0 compaction.
|
|
|
|
// First, clean up memtable before inserting new data. This will generate
|
|
|
|
// a level-0 file, with size around 0.4 (according to previously written
|
|
|
|
// data amount).
|
2014-03-20 17:52:38 +01:00
|
|
|
filter->expect_full_compaction_.store(false);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 3;
|
2013-08-08 00:20:41 +02:00
|
|
|
num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 3);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Generate one more file at level-0, which should trigger level-0
|
|
|
|
// compaction.
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
|
|
|
|
// After comapction, we should have 2 files, with size 4, 2.4.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 2);
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 1; i < options.num_levels ; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Stage 3:
|
|
|
|
// Now we have 2 files at level 0, with size 4 and 2.4. Continue
|
|
|
|
// generating new files at level 0.
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 3;
|
2013-08-08 00:20:41 +02:00
|
|
|
num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 3);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Generate one more file at level-0, which should trigger level-0
|
|
|
|
// compaction.
|
|
|
|
for (int i = 0; i < 12; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Before compaction, we have 4 files at level 0, with size 4, 2.4, 1, 1.
|
|
|
|
// After comapction, we should have 3 files, with size 4, 2.4, 2.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 3);
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 1; i < options.num_levels ; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Stage 4:
|
|
|
|
// Now we have 3 files at level 0, with size 4, 2.4, 2. Let's generate a
|
|
|
|
// new file of size 1.
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Level-0 compaction is triggered, but no file will be picked up.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 4);
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 1; i < options.num_levels ; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Stage 5:
|
|
|
|
// Now we have 4 files at level 0, with size 4, 2.4, 2, 1. Let's generate
|
|
|
|
// a new file of size 1.
|
2014-03-20 00:20:29 +01:00
|
|
|
filter->expect_full_compaction_.store(true);
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-08-08 00:20:41 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// All files at level 0 will be compacted into a single one.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 1; i < options.num_levels ; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-09-10 01:06:10 +02:00
|
|
|
TEST(DBTest, UniversalCompactionSizeAmplification) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-09-10 01:06:10 +02:00
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
2013-09-18 05:02:54 +02:00
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-09-10 01:06:10 +02:00
|
|
|
|
|
|
|
// Trigger compaction if size amplification exceeds 110%
|
2014-02-07 23:47:16 +01:00
|
|
|
options.compaction_options_universal.max_size_amplification_percent = 110;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2013-09-10 01:06:10 +02:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
int key_idx = 0;
|
|
|
|
|
|
|
|
// Generate two files in Level 0. Both files are approx the same size.
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
2013-09-10 01:06:10 +02:00
|
|
|
num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-09-10 01:06:10 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 1);
|
2013-09-10 01:06:10 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 2);
|
2013-09-10 01:06:10 +02:00
|
|
|
|
|
|
|
// Flush whatever is remaining in memtable. This is typically
|
|
|
|
// small, which should not trigger size ratio based compaction
|
|
|
|
// but will instead trigger size amplification.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2013-09-10 01:06:10 +02:00
|
|
|
|
2013-10-16 22:32:53 +02:00
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
|
2013-09-10 01:06:10 +02:00
|
|
|
// Verify that size amplification did occur
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);
|
2013-09-10 01:06:10 +02:00
|
|
|
}
|
|
|
|
|
2013-10-03 01:20:17 +02:00
|
|
|
TEST(DBTest, UniversalCompactionOptions) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-10-03 01:20:17 +02:00
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.level0_file_num_compaction_trigger = 4;
|
|
|
|
options.num_levels = 1;
|
2013-10-17 22:33:39 +02:00
|
|
|
options.compaction_options_universal.compression_size_percent = -1;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-10-03 01:20:17 +02:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
int key_idx = 0;
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
|
2013-10-03 01:20:17 +02:00
|
|
|
key_idx++;
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
|
2013-10-03 01:20:17 +02:00
|
|
|
|
|
|
|
if (num < options.level0_file_num_compaction_trigger - 1) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), num + 1);
|
2013-10-03 01:20:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1);
|
2013-10-03 01:20:17 +02:00
|
|
|
for (int i = 1; i < options.num_levels ; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-10-03 01:20:17 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-12-24 10:09:09 +01:00
|
|
|
TEST(DBTest, UniversalCompactionStopStyleSimilarSize) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
// trigger compaction if there are >= 4 files
|
|
|
|
options.level0_file_num_compaction_trigger = 4;
|
|
|
|
options.compaction_options_universal.size_ratio = 10;
|
|
|
|
options.compaction_options_universal.stop_style = kCompactionStopStyleSimilarSize;
|
2014-01-25 23:12:24 +01:00
|
|
|
options.num_levels=1;
|
2013-12-24 10:09:09 +01:00
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
int key_idx = 0;
|
|
|
|
|
|
|
|
// Stage 1:
|
|
|
|
// Generate a set of files at level 0, but don't trigger level-0
|
|
|
|
// compaction.
|
|
|
|
for (int num = 0;
|
|
|
|
num < options.level0_file_num_compaction_trigger-1;
|
|
|
|
num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-12-24 10:09:09 +01:00
|
|
|
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate one more file at level-0, which should trigger level-0
|
|
|
|
// compaction.
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-12-24 10:09:09 +01:00
|
|
|
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Suppose each file flushed from mem table has size 1. Now we compact
|
|
|
|
// (level0_file_num_compaction_trigger+1)=4 files and should have a big
|
|
|
|
// file of size 4.
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
|
|
|
|
|
|
|
|
// Stage 2:
|
|
|
|
// Now we have one file at level 0, with size 4. We also have some data in
|
|
|
|
// mem table. Let's continue generating new files at level 0, but don't
|
|
|
|
// trigger level-0 compaction.
|
|
|
|
// First, clean up memtable before inserting new data. This will generate
|
|
|
|
// a level-0 file, with size around 0.4 (according to previously written
|
|
|
|
// data amount).
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
for (int num = 0;
|
|
|
|
num < options.level0_file_num_compaction_trigger-3;
|
|
|
|
num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-12-24 10:09:09 +01:00
|
|
|
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), num + 3);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate one more file at level-0, which should trigger level-0
|
|
|
|
// compaction.
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-12-24 10:09:09 +01:00
|
|
|
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
|
|
|
|
// After compaction, we should have 3 files, with size 4, 0.4, 2.
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 3);
|
|
|
|
// Stage 3:
|
|
|
|
// Now we have 3 files at level 0, with size 4, 0.4, 2. Generate one
|
|
|
|
// more file at level-0, which should trigger level-0 compaction.
|
2014-03-13 00:40:14 +01:00
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-12-24 10:09:09 +01:00
|
|
|
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
// Level-0 compaction is triggered, but no file will be picked up.
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 4);
|
|
|
|
}
|
|
|
|
|
2014-04-09 20:43:14 +02:00
|
|
|
#if defined(SNAPPY)
|
2013-11-27 22:32:56 +01:00
|
|
|
TEST(DBTest, CompressedCache) {
|
|
|
|
int num_iter = 80;
|
|
|
|
|
|
|
|
// Run this test three iterations.
|
|
|
|
// Iteration 1: only a uncompressed block cache
|
|
|
|
// Iteration 2: only a compressed block cache
|
|
|
|
// Iteration 3: both block cache and compressed cache
|
2014-04-09 20:43:14 +02:00
|
|
|
// Iteration 4: both block cache and compressed cache, but DB is not
|
|
|
|
// compressed
|
|
|
|
for (int iter = 0; iter < 4; iter++) {
|
2013-11-27 22:32:56 +01:00
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 64*1024; // small write buffer
|
|
|
|
options.statistics = rocksdb::CreateDBStatistics();
|
|
|
|
|
|
|
|
switch (iter) {
|
|
|
|
case 0:
|
|
|
|
// only uncompressed block cache
|
|
|
|
options.block_cache = NewLRUCache(8*1024);
|
|
|
|
options.block_cache_compressed = nullptr;
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
// no block cache, only compressed cache
|
|
|
|
options.no_block_cache = true;
|
|
|
|
options.block_cache = nullptr;
|
|
|
|
options.block_cache_compressed = NewLRUCache(8*1024);
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
// both compressed and uncompressed block cache
|
|
|
|
options.block_cache = NewLRUCache(1024);
|
|
|
|
options.block_cache_compressed = NewLRUCache(8*1024);
|
|
|
|
break;
|
2014-04-09 20:43:14 +02:00
|
|
|
case 3:
|
|
|
|
// both block cache and compressed cache, but DB is not compressed
|
|
|
|
// also, make block cache sizes bigger, to trigger block cache hits
|
|
|
|
options.block_cache = NewLRUCache(1024 * 1024);
|
|
|
|
options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
break;
|
2013-11-27 22:32:56 +01:00
|
|
|
default:
|
|
|
|
ASSERT_TRUE(false);
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2014-02-15 02:47:53 +01:00
|
|
|
// default column family doesn't have block cache
|
|
|
|
Options no_block_cache_opts;
|
|
|
|
no_block_cache_opts.no_block_cache = true;
|
|
|
|
no_block_cache_opts.statistics = options.statistics;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-15 02:47:53 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"},
|
|
|
|
{&no_block_cache_opts, &options});
|
2013-11-27 22:32:56 +01:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Write 8MB (80 values, each 100K)
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
2013-11-27 22:32:56 +01:00
|
|
|
std::vector<std::string> values;
|
|
|
|
std::string str;
|
|
|
|
for (int i = 0; i < num_iter; i++) {
|
|
|
|
if (i % 4 == 0) { // high compression ratio
|
|
|
|
str = RandomString(&rnd, 1000);
|
|
|
|
}
|
|
|
|
values.push_back(str);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), values[i]));
|
2013-11-27 22:32:56 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// flush all data from memtable so that reads are from block cache
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2013-11-27 22:32:56 +01:00
|
|
|
|
|
|
|
for (int i = 0; i < num_iter; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Get(1, Key(i)), values[i]);
|
2013-11-27 22:32:56 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// check that we triggered the appropriate code paths in the cache
|
|
|
|
switch (iter) {
|
|
|
|
case 0:
|
|
|
|
// only uncompressed block cache
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
|
|
|
|
ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
|
2013-11-27 22:32:56 +01:00
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
// no block cache, only compressed cache
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
|
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
|
2013-11-27 22:32:56 +01:00
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
// both compressed and uncompressed block cache
|
2014-01-17 21:46:06 +01:00
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
|
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
|
2013-11-27 22:32:56 +01:00
|
|
|
break;
|
2014-04-09 20:43:14 +02:00
|
|
|
case 3:
|
|
|
|
// both compressed and uncompressed block cache
|
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_MISS), 0);
|
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_HIT), 0);
|
|
|
|
ASSERT_GT(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS), 0);
|
|
|
|
// compressed doesn't have any hits since blocks are not compressed on
|
|
|
|
// storage
|
|
|
|
ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0);
|
|
|
|
break;
|
2013-11-27 22:32:56 +01:00
|
|
|
default:
|
|
|
|
ASSERT_TRUE(false);
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
|
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
2013-11-27 22:32:56 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-11-17 08:44:39 +01:00
|
|
|
static std::string CompressibleString(Random* rnd, int len) {
|
|
|
|
std::string r;
|
|
|
|
test::CompressibleString(rnd, 0.8, len, &r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2013-10-17 22:33:39 +02:00
|
|
|
TEST(DBTest, UniversalCompactionCompressRatio1) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-10-17 22:33:39 +02:00
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.level0_file_num_compaction_trigger = 2;
|
|
|
|
options.num_levels = 1;
|
|
|
|
options.compaction_options_universal.compression_size_percent = 70;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2013-10-17 22:33:39 +02:00
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
int key_idx = 0;
|
|
|
|
|
|
|
|
// The first compaction (2) is compressed.
|
|
|
|
for (int num = 0; num < 2; num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-10-17 22:33:39 +02:00
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
}
|
2014-03-13 00:40:14 +01:00
|
|
|
ASSERT_LT((int)dbfull()->TEST_GetLevel0TotalSize(), 110000 * 2 * 0.9);
|
2013-10-17 22:33:39 +02:00
|
|
|
|
|
|
|
// The second compaction (4) is compressed
|
|
|
|
for (int num = 0; num < 2; num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-10-17 22:33:39 +02:00
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
}
|
2014-03-13 00:40:14 +01:00
|
|
|
ASSERT_LT((int)dbfull()->TEST_GetLevel0TotalSize(), 110000 * 4 * 0.9);
|
2013-10-17 22:33:39 +02:00
|
|
|
|
|
|
|
// The third compaction (2 4) is compressed since this time it is
|
|
|
|
// (1 1 3.2) and 3.2/5.2 doesn't reach ratio.
|
|
|
|
for (int num = 0; num < 2; num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-10-17 22:33:39 +02:00
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
}
|
2014-03-13 00:40:14 +01:00
|
|
|
ASSERT_LT((int)dbfull()->TEST_GetLevel0TotalSize(), 110000 * 6 * 0.9);
|
2013-10-17 22:33:39 +02:00
|
|
|
|
|
|
|
// When we start for the compaction up to (2 4 8), the latest
|
|
|
|
// compressed is not compressed.
|
|
|
|
for (int num = 0; num < 8; num++) {
|
2014-03-13 00:40:14 +01:00
|
|
|
// Write 110KB (11 values, each 10K)
|
|
|
|
for (int i = 0; i < 11; i++) {
|
2013-10-17 22:33:39 +02:00
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
}
|
2014-03-13 00:40:14 +01:00
|
|
|
ASSERT_GT((int)dbfull()->TEST_GetLevel0TotalSize(),
|
|
|
|
110000 * 11 * 0.8 + 110000 * 2);
|
2013-10-17 22:33:39 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, UniversalCompactionCompressRatio2) {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-10-17 22:33:39 +02:00
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.level0_file_num_compaction_trigger = 2;
|
|
|
|
options.num_levels = 1;
|
|
|
|
options.compaction_options_universal.compression_size_percent = 95;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2013-10-17 22:33:39 +02:00
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
int key_idx = 0;
|
|
|
|
|
|
|
|
// When we start for the compaction up to (2 4 8), the latest
|
|
|
|
// compressed is compressed given the size ratio to compress.
|
|
|
|
for (int num = 0; num < 14; num++) {
|
|
|
|
// Write 120KB (12 values, each 10K)
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
ASSERT_OK(Put(Key(key_idx), CompressibleString(&rnd, 10000)));
|
|
|
|
key_idx++;
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForFlushMemTable();
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
}
|
2014-03-13 00:40:14 +01:00
|
|
|
ASSERT_LT((int)dbfull()->TEST_GetLevel0TotalSize(),
|
|
|
|
120000 * 12 * 0.8 + 120000 * 2);
|
2013-10-17 22:33:39 +02:00
|
|
|
}
|
2013-11-17 08:44:39 +01:00
|
|
|
#endif
|
2013-10-17 22:33:39 +02:00
|
|
|
|
2013-09-04 22:13:08 +02:00
|
|
|
TEST(DBTest, ConvertCompactionStyle) {
|
|
|
|
Random rnd(301);
|
|
|
|
int max_key_level_insert = 200;
|
|
|
|
int max_key_universal_insert = 600;
|
|
|
|
|
|
|
|
// Stage 1: generate a db with level compaction
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-09-04 22:13:08 +02:00
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.num_levels = 4;
|
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
|
|
|
options.max_bytes_for_level_base = 500<<10; // 500KB
|
|
|
|
options.max_bytes_for_level_multiplier = 1;
|
|
|
|
options.target_file_size_base = 200<<10; // 200KB
|
|
|
|
options.target_file_size_multiplier = 1;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-09-04 22:13:08 +02:00
|
|
|
|
|
|
|
for (int i = 0; i <= max_key_level_insert; i++) {
|
|
|
|
// each value is 10K
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
|
2013-09-04 22:13:08 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2013-09-04 22:13:08 +02:00
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_GT(TotalTableFiles(1, 4), 1);
|
2013-09-04 22:13:08 +02:00
|
|
|
int non_level0_num_files = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int i = 1; i < options.num_levels; i++) {
|
|
|
|
non_level0_num_files += NumTableFilesAtLevel(i, 1);
|
2013-09-04 22:13:08 +02:00
|
|
|
}
|
|
|
|
ASSERT_GT(non_level0_num_files, 0);
|
|
|
|
|
|
|
|
// Stage 2: reopen with universal compaction - should fail
|
|
|
|
options = CurrentOptions();
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2013-09-04 22:13:08 +02:00
|
|
|
ASSERT_TRUE(s.IsInvalidArgument());
|
|
|
|
|
|
|
|
// Stage 3: compact into a single file and move the file to level 0
|
|
|
|
options = CurrentOptions();
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
options.target_file_size_base = INT_MAX;
|
|
|
|
options.target_file_size_multiplier = 1;
|
|
|
|
options.max_bytes_for_level_base = INT_MAX;
|
|
|
|
options.max_bytes_for_level_multiplier = 1;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2013-09-04 22:13:08 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr, true /* reduce level */,
|
|
|
|
0 /* reduce to level 0 */);
|
2013-09-04 22:13:08 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int i = 0; i < options.num_levels; i++) {
|
|
|
|
int num = NumTableFilesAtLevel(i, 1);
|
2013-09-04 22:13:08 +02:00
|
|
|
if (i == 0) {
|
|
|
|
ASSERT_EQ(num, 1);
|
|
|
|
} else {
|
|
|
|
ASSERT_EQ(num, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Stage 4: re-open in universal compaction style and do some db operations
|
|
|
|
options = CurrentOptions();
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2013-09-04 22:13:08 +02:00
|
|
|
|
|
|
|
for (int i = max_key_level_insert / 2; i <= max_key_universal_insert; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
|
2013-09-04 22:13:08 +02:00
|
|
|
}
|
|
|
|
dbfull()->Flush(FlushOptions());
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2013-09-04 22:13:08 +02:00
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
for (int i = 1; i < options.num_levels; i++) {
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
|
2013-09-04 22:13:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// verify keys inserted in both level compaction style and universal
|
|
|
|
// compaction style
|
|
|
|
std::string keys_in_db;
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = dbfull()->NewIterator(ReadOptions(), handles_[1]);
|
2013-09-04 22:13:08 +02:00
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
|
|
keys_in_db.append(iter->key().ToString());
|
|
|
|
keys_in_db.push_back(',');
|
|
|
|
}
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
std::string expected_keys;
|
|
|
|
for (int i = 0; i <= max_key_universal_insert; i++) {
|
|
|
|
expected_keys.append(Key(i));
|
|
|
|
expected_keys.push_back(',');
|
|
|
|
}
|
|
|
|
|
|
|
|
ASSERT_EQ(keys_in_db, expected_keys);
|
|
|
|
}
|
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2012-11-01 18:50:08 +01:00
|
|
|
void MinLevelHelper(DBTest* self, Options& options) {
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
for (int num = 0;
|
|
|
|
num < options.level0_file_num_compaction_trigger - 1;
|
|
|
|
num++)
|
|
|
|
{
|
|
|
|
std::vector<std::string> values;
|
|
|
|
// Write 120KB (12 values, each 10K)
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
|
|
|
ASSERT_OK(self->Put(Key(i), values[i]));
|
|
|
|
}
|
2013-10-15 00:12:15 +02:00
|
|
|
self->dbfull()->TEST_WaitForFlushMemTable();
|
2012-11-01 18:50:08 +01:00
|
|
|
ASSERT_EQ(self->NumTableFilesAtLevel(0), num + 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
//generate one more file in level-0, and should trigger level-0 compaction
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
|
|
|
ASSERT_OK(self->Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
self->dbfull()->TEST_WaitForCompact();
|
|
|
|
|
|
|
|
ASSERT_EQ(self->NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(self->NumTableFilesAtLevel(1), 1);
|
|
|
|
}
|
|
|
|
|
2012-11-16 21:55:21 +01:00
|
|
|
// returns false if the calling-Test should be skipped
|
|
|
|
bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
|
2012-11-01 18:50:08 +01:00
|
|
|
int lev, int strategy) {
|
2012-11-05 07:04:14 +01:00
|
|
|
fprintf(stderr, "Test with compression options : window_bits = %d, level = %d, strategy = %d}\n", wbits, lev, strategy);
|
2012-11-01 18:50:08 +01:00
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
2012-10-28 07:13:17 +01:00
|
|
|
options.create_if_missing = true;
|
2012-11-01 18:50:08 +01:00
|
|
|
|
|
|
|
if (SnappyCompressionSupported(CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kSnappyCompression;
|
|
|
|
fprintf(stderr, "using snappy\n");
|
|
|
|
} else if (ZlibCompressionSupported(
|
|
|
|
CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kZlibCompression;
|
|
|
|
fprintf(stderr, "using zlib\n");
|
|
|
|
} else if (BZip2CompressionSupported(
|
|
|
|
CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kBZip2Compression;
|
|
|
|
fprintf(stderr, "using bzip2\n");
|
2014-02-08 03:12:30 +01:00
|
|
|
} else if (LZ4CompressionSupported(
|
|
|
|
CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kLZ4Compression;
|
|
|
|
fprintf(stderr, "using lz4\n");
|
|
|
|
} else if (LZ4HCCompressionSupported(
|
|
|
|
CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kLZ4HCCompression;
|
|
|
|
fprintf(stderr, "using lz4hc\n");
|
2012-11-01 18:50:08 +01:00
|
|
|
} else {
|
|
|
|
fprintf(stderr, "skipping test, compression disabled\n");
|
2012-11-16 21:55:21 +01:00
|
|
|
return false;
|
2012-11-01 18:50:08 +01:00
|
|
|
}
|
2013-01-24 19:54:26 +01:00
|
|
|
options.compression_per_level.resize(options.num_levels);
|
2012-10-28 07:13:17 +01:00
|
|
|
|
|
|
|
// do not compress L0
|
|
|
|
for (int i = 0; i < 1; i++) {
|
|
|
|
options.compression_per_level[i] = kNoCompression;
|
|
|
|
}
|
|
|
|
for (int i = 1; i < options.num_levels; i++) {
|
|
|
|
options.compression_per_level[i] = type;
|
|
|
|
}
|
2012-11-16 21:55:21 +01:00
|
|
|
return true;
|
2012-11-01 18:50:08 +01:00
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2013-08-08 00:20:41 +02:00
|
|
|
|
2012-11-01 18:50:08 +01:00
|
|
|
TEST(DBTest, MinLevelToCompress1) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
CompressionType type;
|
2012-11-16 21:55:21 +01:00
|
|
|
if (!MinLevelToCompress(type, options, -14, -1, 0)) {
|
|
|
|
return;
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
Reopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
|
|
|
|
// do not compress L0 and L1
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
options.compression_per_level[i] = kNoCompression;
|
|
|
|
}
|
|
|
|
for (int i = 2; i < options.num_levels; i++) {
|
|
|
|
options.compression_per_level[i] = type;
|
|
|
|
}
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, MinLevelToCompress2) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
CompressionType type;
|
2012-11-16 21:55:21 +01:00
|
|
|
if (!MinLevelToCompress(type, options, 15, -1, 0)) {
|
|
|
|
return;
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
Reopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
|
2012-10-28 07:13:17 +01:00
|
|
|
// do not compress L0 and L1
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
options.compression_per_level[i] = kNoCompression;
|
|
|
|
}
|
|
|
|
for (int i = 2; i < options.num_levels; i++) {
|
|
|
|
options.compression_per_level[i] = type;
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
DestroyAndReopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
}
|
2012-10-28 07:13:17 +01:00
|
|
|
|
2011-07-15 02:20:57 +02:00
|
|
|
TEST(DBTest, RepeatedWritesToSameKey) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000; // Small write buffer
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-07-15 02:20:57 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// We must have at most one file per level except for level-0,
|
|
|
|
// which may have up to kL0_StopWritesTrigger files.
|
2014-02-07 23:47:16 +01:00
|
|
|
const int kMaxFiles =
|
|
|
|
options.num_levels + options.level0_stop_writes_trigger;
|
2011-07-15 02:20:57 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
Random rnd(301);
|
|
|
|
std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
|
|
|
|
for (int i = 0; i < 5 * kMaxFiles; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", value));
|
|
|
|
ASSERT_LE(TotalTableFiles(1), kMaxFiles);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
} while (ChangeCompactOptions());
|
2011-07-15 02:20:57 +02:00
|
|
|
}
|
|
|
|
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
TEST(DBTest, InPlaceUpdate) {
|
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.inplace_update_support = true;
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
|
|
|
|
// Update key with values of smaller size
|
|
|
|
int numValues = 10;
|
|
|
|
for (int i = numValues; i > 0; i--) {
|
|
|
|
std::string value = DummyString(i, 'a');
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", value));
|
|
|
|
ASSERT_EQ(value, Get(1, "key"));
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Only 1 instance for that key.
|
2014-02-07 23:47:16 +01:00
|
|
|
validateNumberOfEntries(1, 1);
|
2014-01-14 16:55:16 +01:00
|
|
|
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, InPlaceUpdateLargeNewValue) {
|
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2014-01-14 16:55:16 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.inplace_update_support = true;
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
|
|
|
|
// Update key with values of larger size
|
2014-01-14 16:55:16 +01:00
|
|
|
int numValues = 10;
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
for (int i = 0; i < numValues; i++) {
|
|
|
|
std::string value = DummyString(i, 'a');
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", value));
|
|
|
|
ASSERT_EQ(value, Get(1, "key"));
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// All 10 updates exist in the internal iterator
|
2014-02-07 23:47:16 +01:00
|
|
|
validateNumberOfEntries(numValues, 1);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
|
2014-01-14 16:55:16 +01:00
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
|
|
|
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
TEST(DBTest, InPlaceUpdateCallbackSmallerSize) {
|
2014-01-14 16:55:16 +01:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2014-01-14 16:55:16 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.inplace_update_support = true;
|
|
|
|
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000;
|
|
|
|
options.inplace_callback =
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
rocksdb::DBTest::updateInPlaceSmallerSize;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2014-01-14 16:55:16 +01:00
|
|
|
|
|
|
|
// Update key with values of smaller size
|
|
|
|
int numValues = 10;
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", DummyString(numValues, 'a')));
|
|
|
|
ASSERT_EQ(DummyString(numValues, 'c'), Get(1, "key"));
|
2014-01-14 16:55:16 +01:00
|
|
|
|
|
|
|
for (int i = numValues; i > 0; i--) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", DummyString(i, 'a')));
|
|
|
|
ASSERT_EQ(DummyString(i - 1, 'b'), Get(1, "key"));
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
}
|
2014-01-14 16:55:16 +01:00
|
|
|
|
|
|
|
// Only 1 instance for that key.
|
2014-02-07 23:47:16 +01:00
|
|
|
validateNumberOfEntries(1, 1);
|
2014-01-14 16:55:16 +01:00
|
|
|
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
TEST(DBTest, InPlaceUpdateCallbackSmallerVarintSize) {
|
2014-01-14 16:55:16 +01:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.inplace_update_support = true;
|
|
|
|
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000;
|
|
|
|
options.inplace_callback =
|
|
|
|
rocksdb::DBTest::updateInPlaceSmallerVarintSize;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
|
|
|
|
// Update key with values of smaller varint size
|
|
|
|
int numValues = 265;
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", DummyString(numValues, 'a')));
|
|
|
|
ASSERT_EQ(DummyString(numValues, 'c'), Get(1, "key"));
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
|
|
|
|
for (int i = numValues; i > 0; i--) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", DummyString(i, 'a')));
|
|
|
|
ASSERT_EQ(DummyString(1, 'b'), Get(1, "key"));
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Only 1 instance for that key.
|
2014-02-07 23:47:16 +01:00
|
|
|
validateNumberOfEntries(1, 1);
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
|
2014-01-14 16:55:16 +01:00
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, InPlaceUpdateCallbackLargeNewValue) {
|
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2014-01-14 16:55:16 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.inplace_update_support = true;
|
|
|
|
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000;
|
|
|
|
options.inplace_callback =
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
rocksdb::DBTest::updateInPlaceLargerSize;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2014-01-14 16:55:16 +01:00
|
|
|
|
|
|
|
// Update key with values of larger size
|
|
|
|
int numValues = 10;
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
for (int i = 0; i < numValues; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", DummyString(i, 'a')));
|
|
|
|
ASSERT_EQ(DummyString(i, 'c'), Get(1, "key"));
|
2014-01-14 16:55:16 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// No inplace updates. All updates are puts with new seq number
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
// All 10 updates exist in the internal iterator
|
2014-02-07 23:47:16 +01:00
|
|
|
validateNumberOfEntries(numValues, 1);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
TEST(DBTest, InPlaceUpdateCallbackNoAction) {
|
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.inplace_update_support = true;
|
|
|
|
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000;
|
|
|
|
options.inplace_callback =
|
|
|
|
rocksdb::DBTest::updateInPlaceNoAction;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
|
Allow callback to change size of existing value. Change return type of the callback function to an enum status to handle 3 cases.
Summary:
This diff fixes 2 hacks:
* The callback function can modify the existing value inplace, if the merged value fits within the existing buffer size. But currently the existing buffer size is not being modified. Now the callback recieves a int* allowing the size to be modified. Since size is encoded as a varint in the internal key for memtable. It might happen that the entire value might have be copied to the new location if the new size varint is smaller than the existing size varint.
* The callback function has 3 functionalities
1. Modify existing buffer inplace, and update size correspondingly. Now to indicate that, Returns 1.
2. Generate a new buffer indicating merged value. Returns 2.
3. Fails to do either of above, based on whatever application logic. Returns 0.
Test Plan: Just make all for now. I'm adding another unit test to test each scenario.
Reviewers: dhruba, haobo
Reviewed By: haobo
CC: leveldb, sdong, kailiu, xinyaohu, sumeet, danguo
Differential Revision: https://reviews.facebook.net/D15195
2014-01-17 00:11:19 +01:00
|
|
|
// Callback function requests no actions from db
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "key", DummyString(1, 'a')));
|
|
|
|
ASSERT_EQ(Get(1, "key"), "NOT_FOUND");
|
In-place updates for equal keys and similar sized values
Summary:
Currently for each put, a fresh memory is allocated, and a new entry is added to the memtable with a new sequence number irrespective of whether the key already exists in the memtable. This diff is an attempt to update the value inplace for existing keys. It currently handles a very simple case:
1. Key already exists in the current memtable. Does not inplace update values in immutable memtable or snapshot
2. Latest value type is a 'put' ie kTypeValue
3. New value size is less than existing value, to avoid reallocating memory
TODO: For a put of an existing key, deallocate memory take by values, for other value types till a kTypeValue is found, ie. remove kTypeMerge.
TODO: Update the transaction log, to allow consistent reload of the memtable.
Test Plan: Added a unit test verifying the inplace update. But some other unit tests broken due to invalid sequence number checks. WIll fix them next.
Reviewers: xinyaohu, sumeet, haobo, dhruba
CC: leveldb
Differential Revision: https://reviews.facebook.net/D12423
Automatic commit by arc
2013-08-19 23:12:47 +02:00
|
|
|
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
2012-10-29 09:13:41 +01:00
|
|
|
TEST(DBTest, CompactionFilter) {
|
|
|
|
Options options = CurrentOptions();
|
2014-03-21 01:32:55 +01:00
|
|
|
options.max_open_files = -1;
|
2012-10-29 09:13:41 +01:00
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
2013-08-13 19:56:20 +02:00
|
|
|
options.compaction_filter_factory = std::make_shared<KeepFilterFactory>();
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-05-04 00:26:12 +02:00
|
|
|
// Write 100K keys, these are written to a few files in L0.
|
2012-10-29 09:13:41 +01:00
|
|
|
const std::string value(10, 'x');
|
2013-05-04 00:26:12 +02:00
|
|
|
for (int i = 0; i < 100000; i++) {
|
2012-10-29 09:13:41 +01:00
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, key, value);
|
2012-10-29 09:13:41 +01:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2012-10-29 09:13:41 +01:00
|
|
|
|
|
|
|
// Push all files to the highest level L2. Verify that
|
|
|
|
// the compaction is each level invokes the filter for
|
|
|
|
// all the keys in that level.
|
|
|
|
cfilter_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
cfilter_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(2, 1), 0);
|
2012-10-29 09:13:41 +01:00
|
|
|
cfilter_count = 0;
|
|
|
|
|
2013-02-15 23:31:24 +01:00
|
|
|
// All the files are in the lowest level.
|
|
|
|
// Verify that all but the 100001st record
|
|
|
|
// has sequence number zero. The 100001st record
|
|
|
|
// is at the tip of this snapshot and cannot
|
|
|
|
// be zeroed out.
|
2013-05-04 00:26:12 +02:00
|
|
|
// TODO: figure out sequence number squashtoo
|
2013-02-15 23:31:24 +01:00
|
|
|
int count = 0;
|
|
|
|
int total = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = dbfull()->TEST_NewInternalIterator(handles_[1]);
|
2013-02-15 23:31:24 +01:00
|
|
|
iter->SeekToFirst();
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(iter->status());
|
2013-02-15 23:31:24 +01:00
|
|
|
while (iter->Valid()) {
|
2013-11-17 22:52:55 +01:00
|
|
|
ParsedInternalKey ikey(Slice(), 0, kTypeValue);
|
2013-02-15 23:31:24 +01:00
|
|
|
ikey.sequence = -1;
|
|
|
|
ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
|
|
|
|
total++;
|
|
|
|
if (ikey.sequence != 0) {
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
iter->Next();
|
|
|
|
}
|
2013-05-04 00:26:12 +02:00
|
|
|
ASSERT_EQ(total, 100000);
|
2013-02-15 23:31:24 +01:00
|
|
|
ASSERT_EQ(count, 1);
|
|
|
|
delete iter;
|
|
|
|
|
2013-05-04 00:26:12 +02:00
|
|
|
// overwrite all the 100K keys once again.
|
|
|
|
for (int i = 0; i < 100000; i++) {
|
2012-10-29 09:13:41 +01:00
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, key, value));
|
2012-10-29 09:13:41 +01:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2012-10-29 09:13:41 +01:00
|
|
|
|
|
|
|
// push all files to the highest level L2. This
|
|
|
|
// means that all keys should pass at least once
|
|
|
|
// via the compaction filter
|
|
|
|
cfilter_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
cfilter_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(2, 1), 0);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
|
|
|
// create a new database with the compaction
|
|
|
|
// filter in such a way that it deletes all keys
|
2013-08-13 19:56:20 +02:00
|
|
|
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
|
2012-10-29 09:13:41 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
|
|
|
// write all the keys once again.
|
2013-05-04 00:26:12 +02:00
|
|
|
for (int i = 0; i < 100000; i++) {
|
2012-10-29 09:13:41 +01:00
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, key, value));
|
2012-10-29 09:13:41 +01:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(2, 1), 0);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
|
|
|
// Push all files to the highest level L2. This
|
|
|
|
// triggers the compaction filter to delete all keys,
|
|
|
|
// verify that at the end of the compaction process,
|
|
|
|
// nothing is left.
|
|
|
|
cfilter_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
cfilter_count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
ASSERT_EQ(cfilter_count, 0);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1, 1), 0);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-05-04 00:26:12 +02:00
|
|
|
// Scan the entire database to ensure that nothing is left
|
2014-02-07 23:47:16 +01:00
|
|
|
iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
iter->SeekToFirst();
|
2013-02-15 23:31:24 +01:00
|
|
|
count = 0;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
count++;
|
|
|
|
iter->Next();
|
|
|
|
}
|
2013-05-04 00:26:12 +02:00
|
|
|
ASSERT_EQ(count, 0);
|
2013-02-15 23:31:24 +01:00
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// The sequence number of the remaining record
|
2013-03-04 19:44:04 +01:00
|
|
|
// is not zeroed out even though it is at the
|
2013-02-15 23:31:24 +01:00
|
|
|
// level Lmax because this record is at the tip
|
2013-05-04 00:26:12 +02:00
|
|
|
// TODO: remove the following or design a different
|
|
|
|
// test
|
2013-02-15 23:31:24 +01:00
|
|
|
count = 0;
|
2014-02-07 23:47:16 +01:00
|
|
|
iter = dbfull()->TEST_NewInternalIterator(handles_[1]);
|
2013-02-15 23:31:24 +01:00
|
|
|
iter->SeekToFirst();
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(iter->status());
|
2012-10-29 09:13:41 +01:00
|
|
|
while (iter->Valid()) {
|
2013-11-17 22:52:55 +01:00
|
|
|
ParsedInternalKey ikey(Slice(), 0, kTypeValue);
|
2013-02-15 23:31:24 +01:00
|
|
|
ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
|
2013-03-19 22:53:22 +01:00
|
|
|
ASSERT_NE(ikey.sequence, (unsigned)0);
|
2012-10-29 09:13:41 +01:00
|
|
|
count++;
|
|
|
|
iter->Next();
|
|
|
|
}
|
2013-05-04 00:26:12 +02:00
|
|
|
ASSERT_EQ(count, 0);
|
2012-10-29 09:13:41 +01:00
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2014-05-08 22:48:39 +02:00
|
|
|
// Tests the edge case where compaction does not produce any output -- all
|
|
|
|
// entries are deleted. The compaction should create bunch of 'DeleteFile'
|
|
|
|
// entries in VersionEdit, but none of the 'AddFile's.
|
|
|
|
TEST(DBTest, CompactionFilterDeletesAll) {
|
|
|
|
Options options;
|
|
|
|
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
// put some data
|
|
|
|
for (int table = 0; table < 4; ++table) {
|
|
|
|
for (int i = 0; i < 10 + table; ++i) {
|
|
|
|
Put(std::to_string(table * 100 + i), "val");
|
|
|
|
}
|
|
|
|
Flush();
|
|
|
|
}
|
|
|
|
|
|
|
|
// this will produce empty file (delete compaction filter)
|
|
|
|
ASSERT_OK(db_->CompactRange(nullptr, nullptr));
|
|
|
|
ASSERT_EQ(0, CountLiveFiles());
|
|
|
|
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Iterator* itr = db_->NewIterator(ReadOptions());
|
|
|
|
itr->SeekToFirst();
|
|
|
|
// empty db
|
|
|
|
ASSERT_TRUE(!itr->Valid());
|
|
|
|
|
|
|
|
delete itr;
|
|
|
|
}
|
|
|
|
|
2012-10-29 09:13:41 +01:00
|
|
|
TEST(DBTest, CompactionFilterWithValueChange) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
2013-08-13 19:56:20 +02:00
|
|
|
options.compaction_filter_factory =
|
2013-12-13 01:36:38 +01:00
|
|
|
std::make_shared<ChangeFilterFactory>();
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Write 100K+1 keys, these are written to a few files
|
|
|
|
// in L0. We do this so that the current snapshot points
|
|
|
|
// to the 100001 key.The compaction filter is not invoked
|
|
|
|
// on keys that are visible via a snapshot because we
|
|
|
|
// anyways cannot delete it.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, key, value);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// push all files to lower levels
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// re-write all data again
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, key, value);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// push all files to lower levels. This should
|
|
|
|
// invoke the compaction filter for all 100000 keys.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
2012-10-29 09:13:41 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// verify that all keys now have the new value that
|
|
|
|
// was set by the compaction process.
|
2014-01-10 02:52:11 +01:00
|
|
|
for (int i = 0; i < 100001; i++) {
|
2013-08-08 00:20:41 +02:00
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
std::string newvalue = Get(1, key);
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
|
|
|
|
}
|
|
|
|
} while (ChangeCompactOptions());
|
2012-10-29 09:13:41 +01:00
|
|
|
}
|
|
|
|
|
2014-03-20 00:20:29 +01:00
|
|
|
TEST(DBTest, CompactionFilterContextManual) {
|
|
|
|
KeepFilterFactory* filter = new KeepFilterFactory();
|
|
|
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
options.compaction_filter_factory.reset(filter);
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
options.level0_file_num_compaction_trigger = 8;
|
|
|
|
Reopen(&options);
|
|
|
|
int num_keys_per_file = 400;
|
|
|
|
for (int j = 0; j < 3; j++) {
|
|
|
|
// Write several keys.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
for (int i = 0; i < num_keys_per_file; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%08d%02d", i, j);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
dbfull()->TEST_FlushMemTable();
|
|
|
|
// Make sure next file is much smaller so automatic compaction will not
|
|
|
|
// be triggered.
|
|
|
|
num_keys_per_file /= 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Force a manual compaction
|
|
|
|
cfilter_count = 0;
|
|
|
|
filter->expect_manual_compaction_.store(true);
|
|
|
|
filter->expect_full_compaction_.store(false); // Manual compaction always
|
|
|
|
// set this flag.
|
|
|
|
dbfull()->CompactRange(nullptr, nullptr);
|
|
|
|
ASSERT_EQ(cfilter_count, 700);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
|
|
|
|
|
|
|
|
// Verify total number of keys is correct after manual compaction.
|
|
|
|
int count = 0;
|
|
|
|
int total = 0;
|
|
|
|
Iterator* iter = dbfull()->TEST_NewInternalIterator();
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
while (iter->Valid()) {
|
|
|
|
ParsedInternalKey ikey(Slice(), 0, kTypeValue);
|
|
|
|
ikey.sequence = -1;
|
|
|
|
ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
|
|
|
|
total++;
|
|
|
|
if (ikey.sequence != 0) {
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
ASSERT_EQ(total, 700);
|
|
|
|
ASSERT_EQ(count, 1);
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2014-01-10 02:52:11 +01:00
|
|
|
class KeepFilterV2 : public CompactionFilterV2 {
|
|
|
|
public:
|
|
|
|
virtual std::vector<bool> Filter(int level,
|
|
|
|
const SliceVector& keys,
|
|
|
|
const SliceVector& existing_values,
|
|
|
|
std::vector<std::string>* new_values,
|
|
|
|
std::vector<bool>* values_changed)
|
|
|
|
const override {
|
|
|
|
cfilter_count++;
|
|
|
|
std::vector<bool> ret;
|
|
|
|
new_values->clear();
|
|
|
|
values_changed->clear();
|
|
|
|
for (unsigned int i = 0; i < keys.size(); ++i) {
|
|
|
|
values_changed->push_back(false);
|
|
|
|
ret.push_back(false);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override {
|
|
|
|
return "KeepFilterV2";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class DeleteFilterV2 : public CompactionFilterV2 {
|
|
|
|
public:
|
|
|
|
virtual std::vector<bool> Filter(int level,
|
|
|
|
const SliceVector& keys,
|
|
|
|
const SliceVector& existing_values,
|
|
|
|
std::vector<std::string>* new_values,
|
|
|
|
std::vector<bool>* values_changed)
|
|
|
|
const override {
|
|
|
|
cfilter_count++;
|
|
|
|
new_values->clear();
|
|
|
|
values_changed->clear();
|
|
|
|
std::vector<bool> ret;
|
|
|
|
for (unsigned int i = 0; i < keys.size(); ++i) {
|
|
|
|
values_changed->push_back(false);
|
|
|
|
ret.push_back(true);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override {
|
|
|
|
return "DeleteFilterV2";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class ChangeFilterV2 : public CompactionFilterV2 {
|
|
|
|
public:
|
|
|
|
virtual std::vector<bool> Filter(int level,
|
|
|
|
const SliceVector& keys,
|
|
|
|
const SliceVector& existing_values,
|
|
|
|
std::vector<std::string>* new_values,
|
|
|
|
std::vector<bool>* values_changed)
|
|
|
|
const override {
|
|
|
|
std::vector<bool> ret;
|
|
|
|
new_values->clear();
|
|
|
|
values_changed->clear();
|
|
|
|
for (unsigned int i = 0; i < keys.size(); ++i) {
|
|
|
|
values_changed->push_back(true);
|
|
|
|
new_values->push_back(NEW_VALUE);
|
|
|
|
ret.push_back(false);
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override {
|
|
|
|
return "ChangeFilterV2";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class KeepFilterFactoryV2 : public CompactionFilterFactoryV2 {
|
|
|
|
public:
|
|
|
|
explicit KeepFilterFactoryV2(const SliceTransform* prefix_extractor)
|
|
|
|
: CompactionFilterFactoryV2(prefix_extractor) { }
|
|
|
|
|
|
|
|
virtual std::unique_ptr<CompactionFilterV2>
|
|
|
|
CreateCompactionFilterV2(
|
|
|
|
const CompactionFilterContext& context) override {
|
|
|
|
return std::unique_ptr<CompactionFilterV2>(new KeepFilterV2());
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override {
|
|
|
|
return "KeepFilterFactoryV2";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class DeleteFilterFactoryV2 : public CompactionFilterFactoryV2 {
|
|
|
|
public:
|
|
|
|
explicit DeleteFilterFactoryV2(const SliceTransform* prefix_extractor)
|
|
|
|
: CompactionFilterFactoryV2(prefix_extractor) { }
|
|
|
|
|
|
|
|
virtual std::unique_ptr<CompactionFilterV2>
|
|
|
|
CreateCompactionFilterV2(
|
|
|
|
const CompactionFilterContext& context) override {
|
|
|
|
return std::unique_ptr<CompactionFilterV2>(new DeleteFilterV2());
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override {
|
|
|
|
return "DeleteFilterFactoryV2";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class ChangeFilterFactoryV2 : public CompactionFilterFactoryV2 {
|
|
|
|
public:
|
|
|
|
explicit ChangeFilterFactoryV2(const SliceTransform* prefix_extractor)
|
|
|
|
: CompactionFilterFactoryV2(prefix_extractor) { }
|
|
|
|
|
|
|
|
virtual std::unique_ptr<CompactionFilterV2>
|
|
|
|
CreateCompactionFilterV2(
|
|
|
|
const CompactionFilterContext& context) override {
|
|
|
|
return std::unique_ptr<CompactionFilterV2>(new ChangeFilterV2());
|
|
|
|
}
|
|
|
|
|
|
|
|
virtual const char* Name() const override {
|
|
|
|
return "ChangeFilterFactoryV2";
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionFilterV2) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
// extract prefix
|
2014-03-29 18:34:47 +01:00
|
|
|
std::unique_ptr<const SliceTransform> prefix_extractor;
|
|
|
|
prefix_extractor.reset(NewFixedPrefixTransform(8));
|
|
|
|
|
2014-01-10 02:52:11 +01:00
|
|
|
options.compaction_filter_factory_v2
|
2014-03-29 18:34:47 +01:00
|
|
|
= std::make_shared<KeepFilterFactoryV2>(prefix_extractor.get());
|
2014-01-10 02:52:11 +01:00
|
|
|
// In a testing environment, we can only flush the application
|
|
|
|
// compaction filter buffer using universal compaction
|
|
|
|
option_config_ = kUniversalCompaction;
|
|
|
|
options.compaction_style = (rocksdb::CompactionStyle)1;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
// Write 100K keys, these are written to a few files in L0.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
for (int i = 0; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%08d%010d", i , i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
|
|
|
|
dbfull()->TEST_FlushMemTable();
|
|
|
|
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr);
|
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr);
|
|
|
|
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
|
|
|
|
|
|
|
|
// All the files are in the lowest level.
|
|
|
|
int count = 0;
|
|
|
|
int total = 0;
|
|
|
|
Iterator* iter = dbfull()->TEST_NewInternalIterator();
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
while (iter->Valid()) {
|
|
|
|
ParsedInternalKey ikey(Slice(), 0, kTypeValue);
|
|
|
|
ikey.sequence = -1;
|
|
|
|
ASSERT_EQ(ParseInternalKey(iter->key(), &ikey), true);
|
|
|
|
total++;
|
|
|
|
if (ikey.sequence != 0) {
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
|
|
|
|
ASSERT_EQ(total, 100000);
|
|
|
|
// 1 snapshot only. Since we are using universal compacton,
|
|
|
|
// the sequence no is cleared for better compression
|
|
|
|
ASSERT_EQ(count, 1);
|
|
|
|
delete iter;
|
|
|
|
|
|
|
|
// create a new database with the compaction
|
|
|
|
// filter in such a way that it deletes all keys
|
|
|
|
options.compaction_filter_factory_v2 =
|
2014-03-29 18:34:47 +01:00
|
|
|
std::make_shared<DeleteFilterFactoryV2>(prefix_extractor.get());
|
2014-01-10 02:52:11 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
// write all the keys once again.
|
|
|
|
for (int i = 0; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%08d%010d", i, i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
|
|
|
|
dbfull()->TEST_FlushMemTable();
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(0), 0);
|
|
|
|
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr);
|
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
|
|
|
|
// Scan the entire database to ensure that nothing is left
|
|
|
|
iter = db_->NewIterator(ReadOptions());
|
|
|
|
iter->SeekToFirst();
|
|
|
|
count = 0;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
count++;
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
|
|
|
|
ASSERT_EQ(count, 0);
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionFilterV2WithValueChange) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
2014-03-29 18:34:47 +01:00
|
|
|
std::unique_ptr<const SliceTransform> prefix_extractor;
|
|
|
|
prefix_extractor.reset(NewFixedPrefixTransform(8));
|
2014-01-10 02:52:11 +01:00
|
|
|
options.compaction_filter_factory_v2 =
|
2014-03-29 18:34:47 +01:00
|
|
|
std::make_shared<ChangeFilterFactoryV2>(prefix_extractor.get());
|
2014-01-10 02:52:11 +01:00
|
|
|
// In a testing environment, we can only flush the application
|
|
|
|
// compaction filter buffer using universal compaction
|
|
|
|
option_config_ = kUniversalCompaction;
|
|
|
|
options.compaction_style = (rocksdb::CompactionStyle)1;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2014-01-10 02:52:11 +01:00
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
// Write 100K+1 keys, these are written to a few files
|
|
|
|
// in L0. We do this so that the current snapshot points
|
|
|
|
// to the 100001 key.The compaction filter is not invoked
|
|
|
|
// on keys that are visible via a snapshot because we
|
|
|
|
// anyways cannot delete it.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%08d%010d", i, i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
|
|
|
|
// push all files to lower levels
|
|
|
|
dbfull()->TEST_FlushMemTable();
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr);
|
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr);
|
|
|
|
|
|
|
|
// verify that all keys now have the new value that
|
|
|
|
// was set by the compaction process.
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%08d%010d", i, i);
|
|
|
|
std::string newvalue = Get(key);
|
|
|
|
ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-03-25 19:34:31 +01:00
|
|
|
TEST(DBTest, CompactionFilterV2NULLPrefix) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
2014-03-29 18:34:47 +01:00
|
|
|
std::unique_ptr<const SliceTransform> prefix_extractor;
|
|
|
|
prefix_extractor.reset(NewFixedPrefixTransform(8));
|
2014-03-25 19:34:31 +01:00
|
|
|
options.compaction_filter_factory_v2 =
|
2014-03-29 18:34:47 +01:00
|
|
|
std::make_shared<ChangeFilterFactoryV2>(prefix_extractor.get());
|
2014-03-25 19:34:31 +01:00
|
|
|
// In a testing environment, we can only flush the application
|
|
|
|
// compaction filter buffer using universal compaction
|
|
|
|
option_config_ = kUniversalCompaction;
|
|
|
|
options.compaction_style = (rocksdb::CompactionStyle)1;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
// Write 100K+1 keys, these are written to a few files
|
|
|
|
// in L0. We do this so that the current snapshot points
|
|
|
|
// to the 100001 key.The compaction filter is not invoked
|
|
|
|
// on keys that are visible via a snapshot because we
|
|
|
|
// anyways cannot delete it.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
char first_key[100];
|
|
|
|
snprintf(first_key, sizeof(first_key), "%s0000%010d", "NULL", 1);
|
|
|
|
Put(first_key, value);
|
|
|
|
for (int i = 1; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "%08d%010d", i, i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
|
|
|
|
char last_key[100];
|
|
|
|
snprintf(last_key, sizeof(last_key), "%s0000%010d", "NULL", 2);
|
|
|
|
Put(last_key, value);
|
|
|
|
|
|
|
|
// push all files to lower levels
|
|
|
|
dbfull()->TEST_FlushMemTable();
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr);
|
|
|
|
|
|
|
|
// verify that all keys now have the new value that
|
|
|
|
// was set by the compaction process.
|
|
|
|
std::string newvalue = Get(first_key);
|
|
|
|
ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
|
|
|
|
newvalue = Get(last_key);
|
|
|
|
ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
|
|
|
|
for (int i = 1; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "%08d%010d", i, i);
|
|
|
|
std::string newvalue = Get(key);
|
|
|
|
ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
TEST(DBTest, SparseMerge) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.compression = kNoCompression;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-03-22 19:32:49 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
FillLevels("A", "Z", 1);
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
// Suppose there is:
|
|
|
|
// small amount of data with prefix A
|
|
|
|
// large amount of data with prefix B
|
|
|
|
// small amount of data with prefix C
|
|
|
|
// and that recent updates have made small changes to all three prefixes.
|
|
|
|
// Check that we do not do a compaction that merges all of B in one shot.
|
|
|
|
const std::string value(1000, 'x');
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "A", "va");
|
2013-08-08 00:20:41 +02:00
|
|
|
// Write approximately 100MB of "B" values
|
|
|
|
for (int i = 0; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, key, value);
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "C", "vc");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
2011-03-22 19:32:49 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Make sparse update
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "A", "va2");
|
|
|
|
Put(1, "B100", "bvalue2");
|
|
|
|
Put(1, "C", "vc2");
|
|
|
|
ASSERT_OK(Flush(1));
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
// Compactions should not cause us to create a situation where
|
|
|
|
// a file overlaps too much data at the next level.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(handles_[1]),
|
|
|
|
20 * 1048576);
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(handles_[1]),
|
|
|
|
20 * 1048576);
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(handles_[1]),
|
|
|
|
20 * 1048576);
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2011-03-22 19:32:49 +01:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
|
|
|
|
bool result = (val >= low) && (val <= high);
|
|
|
|
if (!result) {
|
|
|
|
fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
|
|
|
|
(unsigned long long)(val),
|
|
|
|
(unsigned long long)(low),
|
|
|
|
(unsigned long long)(high));
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, ApproximateSizes) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
Options options;
|
2012-04-17 17:36:46 +02:00
|
|
|
options.write_buffer_size = 100000000; // Large write buffer
|
|
|
|
options.compression = kNoCompression;
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
options = CurrentOptions(options);
|
2012-04-17 17:36:46 +02:00
|
|
|
DestroyAndReopen();
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(Between(Size("", "xyz", 1), 0, 0));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
ASSERT_TRUE(Between(Size("", "xyz", 1), 0, 0));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Write 8MB (80 values, each 100K)
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
2012-04-17 17:36:46 +02:00
|
|
|
const int N = 80;
|
|
|
|
static const int S1 = 100000;
|
|
|
|
static const int S2 = 105000; // Allow some expansion from metadata
|
|
|
|
Random rnd(301);
|
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), RandomString(&rnd, S1)));
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// 0 because GetApproximateSizes() does not account for memtable space
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(Between(Size("", Key(50), 1), 0, 0));
|
2011-04-20 01:01:25 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Check sizes across recovery by reopening a few times
|
|
|
|
for (int run = 0; run < 3; run++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2011-04-21 00:48:11 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
for (int compact_start = 0; compact_start < N; compact_start += 10) {
|
|
|
|
for (int i = 0; i < N; i += 10) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(Between(Size("", Key(i), 1), S1 * i, S2 * i));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(i) + ".suffix", 1), S1 * (i + 1),
|
|
|
|
S2 * (i + 1)));
|
|
|
|
ASSERT_TRUE(Between(Size(Key(i), Key(i + 10), 1), S1 * 10, S2 * 10));
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(Between(Size("", Key(50), 1), S1 * 50, S2 * 50));
|
|
|
|
ASSERT_TRUE(
|
|
|
|
Between(Size("", Key(50) + ".suffix", 1), S1 * 50, S2 * 50));
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
std::string cstart_str = Key(compact_start);
|
|
|
|
std::string cend_str = Key(compact_start + 9);
|
|
|
|
Slice cstart = cstart_str;
|
|
|
|
Slice cend = cend_str;
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(0, &cstart, &cend, handles_[1]);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-04-21 00:48:11 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(1, 1), 0);
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2013-12-20 18:35:24 +01:00
|
|
|
// ApproximateOffsetOf() is not yet implemented in plain table format.
|
2014-05-21 20:43:35 +02:00
|
|
|
} while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction |
|
|
|
|
kSkipPlainTable));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.compression = kNoCompression;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Random rnd(301);
|
|
|
|
std::string big1 = RandomString(&rnd, 100000);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(0), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(1, Key(1), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(1, Key(2), big1));
|
|
|
|
ASSERT_OK(Put(1, Key(3), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(1, Key(4), big1));
|
|
|
|
ASSERT_OK(Put(1, Key(5), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(1, Key(6), RandomString(&rnd, 300000)));
|
|
|
|
ASSERT_OK(Put(1, Key(7), RandomString(&rnd, 10000)));
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
// Check sizes across recovery by reopening a few times
|
|
|
|
for (int run = 0; run < 3; run++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(Between(Size("", Key(0), 1), 0, 0));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(1), 1), 10000, 11000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(2), 1), 20000, 21000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(3), 1), 120000, 121000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(4), 1), 130000, 131000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(5), 1), 230000, 231000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(6), 1), 240000, 241000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(7), 1), 540000, 541000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(8), 1), 550000, 560000));
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(Between(Size(Key(3), Key(5), 1), 110000, 111000));
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2013-12-20 18:35:24 +01:00
|
|
|
// ApproximateOffsetOf() is not yet implemented in plain table format.
|
|
|
|
} while (ChangeOptions(kSkipPlainTable));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IteratorPinsRef) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
Put(1, "foo", "hello");
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Get iterator that will yield the current contents of the DB.
|
2014-02-07 23:47:16 +01:00
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions(), handles_[1]);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Write to force compactions
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "newvalue1");
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < 100; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
// 100K values
|
|
|
|
ASSERT_OK(Put(1, Key(i), Key(i) + std::string(100000, 'v')));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "newvalue2");
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ("foo", iter->key().ToString());
|
|
|
|
ASSERT_EQ("hello", iter->value().ToString());
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
delete iter;
|
|
|
|
} while (ChangeCompactOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, Snapshot) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
Put(0, "foo", "0v1");
|
|
|
|
Put(1, "foo", "1v1");
|
2012-04-17 17:36:46 +02:00
|
|
|
const Snapshot* s1 = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(0, "foo", "0v2");
|
|
|
|
Put(1, "foo", "1v2");
|
2012-04-17 17:36:46 +02:00
|
|
|
const Snapshot* s2 = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(0, "foo", "0v3");
|
|
|
|
Put(1, "foo", "1v3");
|
2012-04-17 17:36:46 +02:00
|
|
|
const Snapshot* s3 = db_->GetSnapshot();
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(0, "foo", "0v4");
|
|
|
|
Put(1, "foo", "1v4");
|
|
|
|
ASSERT_EQ("0v1", Get(0, "foo", s1));
|
|
|
|
ASSERT_EQ("1v1", Get(1, "foo", s1));
|
|
|
|
ASSERT_EQ("0v2", Get(0, "foo", s2));
|
|
|
|
ASSERT_EQ("1v2", Get(1, "foo", s2));
|
|
|
|
ASSERT_EQ("0v3", Get(0, "foo", s3));
|
|
|
|
ASSERT_EQ("1v3", Get(1, "foo", s3));
|
|
|
|
ASSERT_EQ("0v4", Get(0, "foo"));
|
|
|
|
ASSERT_EQ("1v4", Get(1, "foo"));
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
db_->ReleaseSnapshot(s3);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("0v1", Get(0, "foo", s1));
|
|
|
|
ASSERT_EQ("1v1", Get(1, "foo", s1));
|
|
|
|
ASSERT_EQ("0v2", Get(0, "foo", s2));
|
|
|
|
ASSERT_EQ("1v2", Get(1, "foo", s2));
|
|
|
|
ASSERT_EQ("0v4", Get(0, "foo"));
|
|
|
|
ASSERT_EQ("1v4", Get(1, "foo"));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
db_->ReleaseSnapshot(s1);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("0v2", Get(0, "foo", s2));
|
|
|
|
ASSERT_EQ("1v2", Get(1, "foo", s2));
|
|
|
|
ASSERT_EQ("0v4", Get(0, "foo"));
|
|
|
|
ASSERT_EQ("1v4", Get(1, "foo"));
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
db_->ReleaseSnapshot(s2);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("0v4", Get(0, "foo"));
|
|
|
|
ASSERT_EQ("1v4", Get(1, "foo"));
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
} while (ChangeOptions(kSkipHashCuckoo));
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
TEST(DBTest, HiddenValuesAreRemoved) {
|
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-04-17 17:36:46 +02:00
|
|
|
Random rnd(301);
|
2014-02-07 23:47:16 +01:00
|
|
|
FillLevels("a", "z", 1);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
std::string big = RandomString(&rnd, 50000);
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", big);
|
|
|
|
Put(1, "pastfoo", "v");
|
2012-04-17 17:36:46 +02:00
|
|
|
const Snapshot* snapshot = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "tiny");
|
|
|
|
Put(1, "pastfoo2", "v2"); // Advance sequence number one more
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(0, 1), 0);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(big, Get(1, "foo", snapshot));
|
|
|
|
ASSERT_TRUE(Between(Size("", "pastfoo", 1), 50000, 60000));
|
2012-04-17 17:36:46 +02:00
|
|
|
db_->ReleaseSnapshot(snapshot);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny, " + big + " ]");
|
2012-04-17 17:36:46 +02:00
|
|
|
Slice x("x");
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(0, nullptr, &x, handles_[1]);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny ]");
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
|
|
|
ASSERT_GE(NumTableFilesAtLevel(1, 1), 1);
|
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, &x, handles_[1]);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ tiny ]");
|
|
|
|
|
|
|
|
ASSERT_TRUE(Between(Size("", "pastfoo", 1), 0, 1000));
|
2013-12-20 18:35:24 +01:00
|
|
|
// ApproximateOffsetOf() is not yet implemented in plain table format,
|
|
|
|
// which is used by Size().
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// skip HashCuckooRep as it does not support snapshot
|
2014-05-21 20:43:35 +02:00
|
|
|
} while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction |
|
|
|
|
kSkipPlainTable | kSkipHashCuckoo));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-11-27 06:16:21 +01:00
|
|
|
TEST(DBTest, CompactBetweenSnapshots) {
|
|
|
|
do {
|
2014-04-25 21:23:07 +02:00
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.disable_auto_compactions = true;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-11-27 06:16:21 +01:00
|
|
|
Random rnd(301);
|
2014-02-07 23:47:16 +01:00
|
|
|
FillLevels("a", "z", 1);
|
2012-11-27 06:16:21 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "first");
|
2012-11-27 06:16:21 +01:00
|
|
|
const Snapshot* snapshot1 = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "second");
|
|
|
|
Put(1, "foo", "third");
|
|
|
|
Put(1, "foo", "fourth");
|
2012-11-27 06:16:21 +01:00
|
|
|
const Snapshot* snapshot2 = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "fifth");
|
|
|
|
Put(1, "foo", "sixth");
|
2012-11-27 06:16:21 +01:00
|
|
|
|
|
|
|
// All entries (including duplicates) exist
|
|
|
|
// before any compaction is triggered.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ("sixth", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
|
|
|
|
ASSERT_EQ("first", Get(1, "foo", snapshot1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1),
|
2012-11-27 06:16:21 +01:00
|
|
|
"[ sixth, fifth, fourth, third, second, first ]");
|
|
|
|
|
|
|
|
// After a compaction, "second", "third" and "fifth" should
|
|
|
|
// be removed
|
2014-02-07 23:47:16 +01:00
|
|
|
FillLevels("a", "z", 1);
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ("sixth", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
|
|
|
|
ASSERT_EQ("first", Get(1, "foo", snapshot1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]");
|
2012-11-27 06:16:21 +01:00
|
|
|
|
|
|
|
// after we release the snapshot1, only two values left
|
|
|
|
db_->ReleaseSnapshot(snapshot1);
|
2014-02-07 23:47:16 +01:00
|
|
|
FillLevels("a", "z", 1);
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
2012-11-27 06:16:21 +01:00
|
|
|
|
|
|
|
// We have only one valid snapshot snapshot2. Since snapshot1 is
|
|
|
|
// not valid anymore, "first" should be removed by a compaction.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("sixth", Get(1, "foo"));
|
|
|
|
ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]");
|
2012-11-27 06:16:21 +01:00
|
|
|
|
|
|
|
// after we release the snapshot2, only one value should be left
|
|
|
|
db_->ReleaseSnapshot(snapshot2);
|
2014-02-07 23:47:16 +01:00
|
|
|
FillLevels("a", "z", 1);
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ("sixth", Get(1, "foo"));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// skip HashCuckooRep as it does not support snapshot
|
2014-05-21 20:43:35 +02:00
|
|
|
} while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
|
2012-11-27 06:16:21 +01:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, DeletionMarkers1) {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
Put(1, "foo", "v1");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
const int last = CurrentOptions().max_mem_compaction_level;
|
|
|
|
// foo => v1 is now in last level
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
// Place a table at level last-1 to prevent merging with preceding mutation
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "a", "begin");
|
|
|
|
Put(1, "z", "end");
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last - 1, 1), 1);
|
|
|
|
|
|
|
|
Delete(1, "foo");
|
|
|
|
Put(1, "foo", "v2");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
|
|
|
|
ASSERT_OK(Flush(1)); // Moves to level last-2
|
2013-02-28 23:09:30 +01:00
|
|
|
if (CurrentOptions().purge_redundant_kvs_while_flush) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
} else {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
}
|
2011-10-06 01:30:28 +02:00
|
|
|
Slice z("z");
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(last - 2, nullptr, &z, handles_[1]);
|
2011-03-18 23:37:00 +01:00
|
|
|
// DEL eliminated, but v1 remains because we aren't compacting that level
|
|
|
|
// (DEL can be eliminated because v2 hides v1).
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
|
|
|
|
dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr, handles_[1]);
|
2011-06-22 04:36:45 +02:00
|
|
|
// Merging last-1 w/ last, so we are the base level for "foo", so
|
|
|
|
// DEL is removed. (as is v1).
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, DeletionMarkers2) {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
Put(1, "foo", "v1");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
const int last = CurrentOptions().max_mem_compaction_level;
|
|
|
|
// foo => v1 is now in last level
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
|
|
|
// Place a table at level last-1 to prevent merging with preceding mutation
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "a", "begin");
|
|
|
|
Put(1, "z", "end");
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last, 1), 1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last - 1, 1), 1);
|
|
|
|
|
|
|
|
Delete(1, "foo");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]");
|
|
|
|
ASSERT_OK(Flush(1)); // Moves to level last-2
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]");
|
|
|
|
dbfull()->TEST_CompactRange(last - 2, nullptr, nullptr, handles_[1]);
|
2011-06-22 04:36:45 +02:00
|
|
|
// DEL kept: "last" file overlaps
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v1 ]");
|
|
|
|
dbfull()->TEST_CompactRange(last - 1, nullptr, nullptr, handles_[1]);
|
2011-06-22 04:36:45 +02:00
|
|
|
// Merging last-1 w/ last, so we are the base level for "foo", so
|
|
|
|
// DEL is removed. (as is v1).
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
TEST(DBTest, OverlapInLevel0) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
int tmp = CurrentOptions().max_mem_compaction_level;
|
2012-06-23 04:30:03 +02:00
|
|
|
ASSERT_EQ(tmp, 2) << "Fix test to match config";
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2013-10-04 19:21:03 +02:00
|
|
|
//Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "100", "v100"));
|
|
|
|
ASSERT_OK(Put(1, "999", "v999"));
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_OK(Delete(1, "100"));
|
|
|
|
ASSERT_OK(Delete(1, "999"));
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_EQ("0,1,1", FilesPerLevel(1));
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
// Make files spanning the following ranges in level-0:
|
|
|
|
// files[0] 200 .. 900
|
|
|
|
// files[1] 300 .. 500
|
|
|
|
// Note that files are sorted by smallest key.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "300", "v300"));
|
|
|
|
ASSERT_OK(Put(1, "500", "v500"));
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_OK(Put(1, "200", "v200"));
|
|
|
|
ASSERT_OK(Put(1, "600", "v600"));
|
|
|
|
ASSERT_OK(Put(1, "900", "v900"));
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_EQ("2,1,1", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Compact away the placeholder files we created initially
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
|
|
|
|
dbfull()->TEST_CompactRange(2, nullptr, nullptr, handles_[1]);
|
|
|
|
ASSERT_EQ("2", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Do a memtable compaction. Before bug-fix, the compaction would
|
|
|
|
// not detect the overlap with level-0 files and would incorrectly place
|
|
|
|
// the deletion in a deeper level.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "600"));
|
|
|
|
Flush(1);
|
|
|
|
ASSERT_EQ("3", FilesPerLevel(1));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, "600"));
|
2014-05-21 20:43:35 +02:00
|
|
|
} while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction));
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
TEST(DBTest, L0_CompactionBug_Issue44_a) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "b", "v"));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_OK(Delete(1, "b"));
|
|
|
|
ASSERT_OK(Delete(1, "a"));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_OK(Delete(1, "a"));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "a", "v"));
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("(a->v)", Contents(1));
|
2013-08-08 00:20:41 +02:00
|
|
|
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("(a->v)", Contents(1));
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, L0_CompactionBug_Issue44_b) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
Put(1, "", "");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Delete(1, "e");
|
|
|
|
Put(1, "", "");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Put(1, "c", "cv");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Put(1, "", "");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Put(1, "", "");
|
2013-08-08 00:20:41 +02:00
|
|
|
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Put(1, "d", "dv");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Put(1, "", "");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
Delete(1, "d");
|
|
|
|
Delete(1, "b");
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"});
|
|
|
|
ASSERT_EQ("(->)(c->cv)", Contents(1));
|
2013-08-08 00:20:41 +02:00
|
|
|
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("(->)(c->cv)", Contents(1));
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, ComparatorCheck) {
|
|
|
|
class NewComparator : public Comparator {
|
|
|
|
public:
|
2013-10-05 07:32:05 +02:00
|
|
|
virtual const char* Name() const { return "rocksdb.NewComparator"; }
|
2011-03-18 23:37:00 +01:00
|
|
|
virtual int Compare(const Slice& a, const Slice& b) const {
|
|
|
|
return BytewiseComparator()->Compare(a, b);
|
|
|
|
}
|
|
|
|
virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
|
|
|
|
BytewiseComparator()->FindShortestSeparator(s, l);
|
|
|
|
}
|
|
|
|
virtual void FindShortSuccessor(std::string* key) const {
|
|
|
|
BytewiseComparator()->FindShortSuccessor(key);
|
|
|
|
}
|
|
|
|
};
|
2014-02-07 23:47:16 +01:00
|
|
|
Options new_options, options;
|
2013-10-01 23:46:52 +02:00
|
|
|
NewComparator cmp;
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
options = CurrentOptions();
|
2013-10-01 23:46:52 +02:00
|
|
|
new_options = CurrentOptions();
|
2013-08-08 00:20:41 +02:00
|
|
|
new_options.comparator = &cmp;
|
2014-02-07 23:47:16 +01:00
|
|
|
// only the non-default column family has non-matching comparator
|
|
|
|
Status s = TryReopenWithColumnFamilies({"default", "pikachu"},
|
|
|
|
{&options, &new_options});
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
|
|
|
|
<< s.ToString();
|
2013-10-01 23:46:52 +02:00
|
|
|
} while (ChangeCompactOptions(&new_options));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
TEST(DBTest, CustomComparator) {
|
|
|
|
class NumberComparator : public Comparator {
|
|
|
|
public:
|
|
|
|
virtual const char* Name() const { return "test.NumberComparator"; }
|
|
|
|
virtual int Compare(const Slice& a, const Slice& b) const {
|
2011-11-14 18:06:16 +01:00
|
|
|
return ToNumber(a) - ToNumber(b);
|
|
|
|
}
|
|
|
|
virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
|
|
|
|
ToNumber(*s); // Check format
|
|
|
|
ToNumber(l); // Check format
|
|
|
|
}
|
|
|
|
virtual void FindShortSuccessor(std::string* key) const {
|
|
|
|
ToNumber(*key); // Check format
|
|
|
|
}
|
|
|
|
private:
|
|
|
|
static int ToNumber(const Slice& x) {
|
|
|
|
// Check that there are no extra characters.
|
|
|
|
ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']')
|
|
|
|
<< EscapeString(x);
|
|
|
|
int val;
|
|
|
|
char ignored;
|
|
|
|
ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1)
|
|
|
|
<< EscapeString(x);
|
|
|
|
return val;
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
};
|
2013-10-01 23:46:52 +02:00
|
|
|
Options new_options;
|
|
|
|
NumberComparator cmp;
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2013-10-01 23:46:52 +02:00
|
|
|
new_options = CurrentOptions();
|
2013-08-08 00:20:41 +02:00
|
|
|
new_options.create_if_missing = true;
|
|
|
|
new_options.comparator = &cmp;
|
|
|
|
new_options.filter_policy = nullptr; // Cannot use bloom filters
|
|
|
|
new_options.write_buffer_size = 1000; // Compact more often
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
new_options = CurrentOptions(new_options);
|
2013-08-08 00:20:41 +02:00
|
|
|
DestroyAndReopen(&new_options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &new_options);
|
|
|
|
ASSERT_OK(Put(1, "[10]", "ten"));
|
|
|
|
ASSERT_OK(Put(1, "[0x14]", "twenty"));
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < 2; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("ten", Get(1, "[10]"));
|
|
|
|
ASSERT_EQ("ten", Get(1, "[0xa]"));
|
|
|
|
ASSERT_EQ("twenty", Get(1, "[20]"));
|
|
|
|
ASSERT_EQ("twenty", Get(1, "[0x14]"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, "[15]"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, "[0xf]"));
|
|
|
|
Compact(1, "[0]", "[9999]");
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
for (int run = 0; run < 2; run++) {
|
|
|
|
for (int i = 0; i < 1000; i++) {
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "[%d]", i*10);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, buf, buf));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
Compact(1, "[0]", "[1000000]");
|
2011-11-14 18:06:16 +01:00
|
|
|
}
|
2013-10-01 23:46:52 +02:00
|
|
|
} while (ChangeCompactOptions(&new_options));
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
TEST(DBTest, ManualCompaction) {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2012-06-23 04:30:03 +02:00
|
|
|
ASSERT_EQ(dbfull()->MaxMemCompactionLevel(), 2)
|
2011-10-06 01:30:28 +02:00
|
|
|
<< "Need to update this test to match kMaxMemCompactLevel";
|
|
|
|
|
2014-01-15 01:19:09 +01:00
|
|
|
// iter - 0 with 7 levels
|
|
|
|
// iter - 1 with 3 levels
|
|
|
|
for (int iter = 0; iter < 2; ++iter) {
|
2014-02-07 23:47:16 +01:00
|
|
|
MakeTables(3, "p", "q", 1);
|
|
|
|
ASSERT_EQ("1,1,1", FilesPerLevel(1));
|
2014-01-15 01:19:09 +01:00
|
|
|
|
|
|
|
// Compaction range falls before files
|
2014-02-07 23:47:16 +01:00
|
|
|
Compact(1, "", "c");
|
|
|
|
ASSERT_EQ("1,1,1", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2014-01-15 01:19:09 +01:00
|
|
|
// Compaction range falls after files
|
2014-02-07 23:47:16 +01:00
|
|
|
Compact(1, "r", "z");
|
|
|
|
ASSERT_EQ("1,1,1", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2014-01-15 01:19:09 +01:00
|
|
|
// Compaction range overlaps files
|
2014-02-07 23:47:16 +01:00
|
|
|
Compact(1, "p1", "p9");
|
|
|
|
ASSERT_EQ("0,0,1", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2014-01-15 01:19:09 +01:00
|
|
|
// Populate a different range
|
2014-02-07 23:47:16 +01:00
|
|
|
MakeTables(3, "c", "e", 1);
|
|
|
|
ASSERT_EQ("1,1,2", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2014-01-15 01:19:09 +01:00
|
|
|
// Compact just the new range
|
2014-02-07 23:47:16 +01:00
|
|
|
Compact(1, "b", "f");
|
|
|
|
ASSERT_EQ("0,0,2", FilesPerLevel(1));
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2014-01-15 01:19:09 +01:00
|
|
|
// Compact all
|
2014-02-07 23:47:16 +01:00
|
|
|
MakeTables(1, "a", "z", 1);
|
|
|
|
ASSERT_EQ("0,1,2", FilesPerLevel(1));
|
|
|
|
db_->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ("0,0,1", FilesPerLevel(1));
|
2014-01-15 01:19:09 +01:00
|
|
|
|
|
|
|
if (iter == 0) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2014-01-15 01:19:09 +01:00
|
|
|
}
|
|
|
|
}
|
2011-10-06 01:30:28 +02:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, DBOpen_Options) {
|
|
|
|
std::string dbname = test::TmpDir() + "/db_options_test";
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(DestroyDB(dbname, Options()));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Does not exist, and create_if_missing == false: error
|
2013-03-01 03:04:58 +01:00
|
|
|
DB* db = nullptr;
|
2011-03-18 23:37:00 +01:00
|
|
|
Options opts;
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
Status s = DB::Open(opts, dbname, &db);
|
2013-03-01 03:04:58 +01:00
|
|
|
ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
|
|
|
|
ASSERT_TRUE(db == nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Does not exist, and create_if_missing == true: OK
|
|
|
|
opts.create_if_missing = true;
|
|
|
|
s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_OK(s);
|
2013-03-01 03:04:58 +01:00
|
|
|
ASSERT_TRUE(db != nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
delete db;
|
2013-03-01 03:04:58 +01:00
|
|
|
db = nullptr;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Does exist, and error_if_exists == true: error
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
opts.error_if_exists = true;
|
|
|
|
s = DB::Open(opts, dbname, &db);
|
2013-03-01 03:04:58 +01:00
|
|
|
ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
|
|
|
|
ASSERT_TRUE(db == nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// Does exist, and error_if_exists == false: OK
|
|
|
|
opts.create_if_missing = true;
|
|
|
|
opts.error_if_exists = false;
|
|
|
|
s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_OK(s);
|
2013-03-01 03:04:58 +01:00
|
|
|
ASSERT_TRUE(db != nullptr);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
delete db;
|
2013-03-01 03:04:58 +01:00
|
|
|
db = nullptr;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-10-29 23:25:01 +01:00
|
|
|
TEST(DBTest, DBOpen_Change_NumLevels) {
|
|
|
|
Options opts;
|
|
|
|
opts.create_if_missing = true;
|
2014-02-07 23:47:16 +01:00
|
|
|
DestroyAndReopen(&opts);
|
|
|
|
ASSERT_TRUE(db_ != nullptr);
|
|
|
|
CreateAndReopenWithCF({"pikachu"}, &opts);
|
|
|
|
|
|
|
|
ASSERT_OK(Put(1, "a", "123"));
|
|
|
|
ASSERT_OK(Put(1, "b", "234"));
|
|
|
|
db_->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
Close();
|
2012-10-29 23:25:01 +01:00
|
|
|
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
opts.num_levels = 2;
|
2014-02-07 23:47:16 +01:00
|
|
|
Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, &opts);
|
2014-01-15 00:54:11 +01:00
|
|
|
ASSERT_TRUE(strstr(s.ToString().c_str(), "Invalid argument") != nullptr);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_TRUE(db_ == nullptr);
|
2012-10-29 23:25:01 +01:00
|
|
|
}
|
|
|
|
|
2012-12-17 20:26:59 +01:00
|
|
|
TEST(DBTest, DestroyDBMetaDatabase) {
|
|
|
|
std::string dbname = test::TmpDir() + "/db_meta";
|
|
|
|
std::string metadbname = MetaDatabaseName(dbname, 0);
|
|
|
|
std::string metametadbname = MetaDatabaseName(metadbname, 0);
|
|
|
|
|
|
|
|
// Destroy previous versions if they exist. Using the long way.
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(DestroyDB(metametadbname, Options()));
|
|
|
|
ASSERT_OK(DestroyDB(metadbname, Options()));
|
|
|
|
ASSERT_OK(DestroyDB(dbname, Options()));
|
2012-12-17 20:26:59 +01:00
|
|
|
|
|
|
|
// Setup databases
|
|
|
|
Options opts;
|
|
|
|
opts.create_if_missing = true;
|
2013-03-01 03:04:58 +01:00
|
|
|
DB* db = nullptr;
|
2012-12-17 20:26:59 +01:00
|
|
|
ASSERT_OK(DB::Open(opts, dbname, &db));
|
|
|
|
delete db;
|
2013-03-01 03:04:58 +01:00
|
|
|
db = nullptr;
|
2012-12-17 20:26:59 +01:00
|
|
|
ASSERT_OK(DB::Open(opts, metadbname, &db));
|
|
|
|
delete db;
|
2013-03-01 03:04:58 +01:00
|
|
|
db = nullptr;
|
2012-12-17 20:26:59 +01:00
|
|
|
ASSERT_OK(DB::Open(opts, metametadbname, &db));
|
|
|
|
delete db;
|
2013-03-01 03:04:58 +01:00
|
|
|
db = nullptr;
|
2012-12-17 20:26:59 +01:00
|
|
|
|
|
|
|
// Delete databases
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(DestroyDB(dbname, Options()));
|
2012-12-17 20:26:59 +01:00
|
|
|
|
|
|
|
// Check if deletion worked.
|
|
|
|
opts.create_if_missing = false;
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_TRUE(!(DB::Open(opts, dbname, &db)).ok());
|
|
|
|
ASSERT_TRUE(!(DB::Open(opts, metadbname, &db)).ok());
|
|
|
|
ASSERT_TRUE(!(DB::Open(opts, metametadbname, &db)).ok());
|
2012-12-17 20:26:59 +01:00
|
|
|
}
|
|
|
|
|
2012-01-25 23:56:52 +01:00
|
|
|
// Check that number of files does not grow when we are out of space
|
|
|
|
TEST(DBTest, NoSpace) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
2014-03-31 21:44:54 +02:00
|
|
|
options.paranoid_checks = false;
|
2013-08-08 00:20:41 +02:00
|
|
|
Reopen(&options);
|
2012-01-25 23:56:52 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
Compact("a", "z");
|
|
|
|
const int num_files = CountFiles();
|
|
|
|
env_->no_space_.Release_Store(env_); // Force out-of-space errors
|
|
|
|
env_->sleep_counter_.Reset();
|
|
|
|
for (int i = 0; i < 5; i++) {
|
|
|
|
for (int level = 0; level < dbfull()->NumberLevels()-1; level++) {
|
|
|
|
dbfull()->TEST_CompactRange(level, nullptr, nullptr);
|
|
|
|
}
|
2012-01-25 23:56:52 +01:00
|
|
|
}
|
2014-03-18 20:25:08 +01:00
|
|
|
|
|
|
|
std::string property_value;
|
|
|
|
ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
|
|
|
|
ASSERT_EQ("5", property_value);
|
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
env_->no_space_.Release_Store(nullptr);
|
|
|
|
ASSERT_LT(CountFiles(), num_files + 3);
|
2012-08-23 01:57:51 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Check that compaction attempts slept after errors
|
|
|
|
ASSERT_GE(env_->sleep_counter_.Read(), 5);
|
|
|
|
} while (ChangeCompactOptions());
|
2012-08-23 01:57:51 +02:00
|
|
|
}
|
|
|
|
|
2014-03-18 20:25:08 +01:00
|
|
|
// Check background error counter bumped on flush failures.
|
|
|
|
TEST(DBTest, NoSpaceFlush) {
|
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.max_background_flushes = 1;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
env_->no_space_.Release_Store(env_); // Force out-of-space errors
|
|
|
|
|
|
|
|
std::string property_value;
|
|
|
|
// Background error count is 0 now.
|
|
|
|
ASSERT_TRUE(db_->GetProperty("rocksdb.background-errors", &property_value));
|
|
|
|
ASSERT_EQ("0", property_value);
|
|
|
|
|
|
|
|
dbfull()->TEST_FlushMemTable(false);
|
|
|
|
|
|
|
|
// Wait 300 milliseconds or background-errors turned 1 from 0.
|
|
|
|
int time_to_sleep_limit = 300000;
|
|
|
|
while (time_to_sleep_limit > 0) {
|
|
|
|
int to_sleep = (time_to_sleep_limit > 1000) ? 1000 : time_to_sleep_limit;
|
|
|
|
time_to_sleep_limit -= to_sleep;
|
|
|
|
env_->SleepForMicroseconds(to_sleep);
|
|
|
|
|
|
|
|
ASSERT_TRUE(
|
|
|
|
db_->GetProperty("rocksdb.background-errors", &property_value));
|
|
|
|
if (property_value == "1") {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ASSERT_EQ("1", property_value);
|
|
|
|
|
|
|
|
env_->no_space_.Release_Store(nullptr);
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
TEST(DBTest, NonWritableFileSystem) {
|
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 1000;
|
|
|
|
options.env = env_;
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
env_->non_writable_.Release_Store(env_); // Force errors for new files
|
|
|
|
std::string big(100000, 'x');
|
|
|
|
int errors = 0;
|
|
|
|
for (int i = 0; i < 20; i++) {
|
|
|
|
if (!Put("foo", big).ok()) {
|
|
|
|
errors++;
|
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
2012-08-23 01:57:51 +02:00
|
|
|
}
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_GT(errors, 0);
|
|
|
|
env_->non_writable_.Release_Store(nullptr);
|
|
|
|
} while (ChangeCompactOptions());
|
2012-01-25 23:56:52 +01:00
|
|
|
}
|
|
|
|
|
2013-01-08 21:00:13 +01:00
|
|
|
TEST(DBTest, ManifestWriteError) {
|
|
|
|
// Test for the following problem:
|
|
|
|
// (a) Compaction produces file F
|
|
|
|
// (b) Log record containing F is written to MANIFEST file, but Sync() fails
|
|
|
|
// (c) GC deletes F
|
|
|
|
// (d) After reopening DB, reads fail since deleted F is named in log record
|
|
|
|
|
|
|
|
// We iterate twice. In the second iteration, everything is the
|
|
|
|
// same except the log record never makes it to the MANIFEST file.
|
|
|
|
for (int iter = 0; iter < 2; iter++) {
|
|
|
|
port::AtomicPointer* error_type = (iter == 0)
|
|
|
|
? &env_->manifest_sync_error_
|
|
|
|
: &env_->manifest_write_error_;
|
|
|
|
|
|
|
|
// Insert foo=>bar mapping
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.error_if_exists = false;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
ASSERT_OK(Put("foo", "bar"));
|
|
|
|
ASSERT_EQ("bar", Get("foo"));
|
|
|
|
|
|
|
|
// Memtable compaction (will succeed)
|
2014-02-07 23:47:16 +01:00
|
|
|
Flush();
|
2013-01-08 21:00:13 +01:00
|
|
|
ASSERT_EQ("bar", Get("foo"));
|
|
|
|
const int last = dbfull()->MaxMemCompactionLevel();
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo=>bar is now in last level
|
|
|
|
|
|
|
|
// Merging compaction (will fail)
|
|
|
|
error_type->Release_Store(env_);
|
2013-03-01 03:04:58 +01:00
|
|
|
dbfull()->TEST_CompactRange(last, nullptr, nullptr); // Should fail
|
2013-01-08 21:00:13 +01:00
|
|
|
ASSERT_EQ("bar", Get("foo"));
|
|
|
|
|
|
|
|
// Recovery: should not lose data
|
2013-03-01 03:04:58 +01:00
|
|
|
error_type->Release_Store(nullptr);
|
2013-01-08 21:00:13 +01:00
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ("bar", Get("foo"));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-10-28 20:36:02 +01:00
|
|
|
TEST(DBTest, PutFailsParanoid) {
|
|
|
|
// Test the following:
|
|
|
|
// (a) A random put fails in paranoid mode (simulate by sync fail)
|
|
|
|
// (b) All other puts have to fail, even if writes would succeed
|
|
|
|
// (c) All of that should happen ONLY if paranoid_checks = true
|
|
|
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.error_if_exists = false;
|
|
|
|
options.paranoid_checks = true;
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-10-28 20:36:02 +01:00
|
|
|
Status s;
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "bar"));
|
|
|
|
ASSERT_OK(Put(1, "foo1", "bar1"));
|
2013-10-28 20:36:02 +01:00
|
|
|
// simulate error
|
|
|
|
env_->log_write_error_.Release_Store(env_);
|
2014-02-07 23:47:16 +01:00
|
|
|
s = Put(1, "foo2", "bar2");
|
2013-10-28 20:36:02 +01:00
|
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
env_->log_write_error_.Release_Store(nullptr);
|
2014-02-07 23:47:16 +01:00
|
|
|
s = Put(1, "foo3", "bar3");
|
2013-10-28 20:36:02 +01:00
|
|
|
// the next put should fail, too
|
|
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
// but we're still able to read
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("bar", Get(1, "foo"));
|
2013-10-28 20:36:02 +01:00
|
|
|
|
|
|
|
// do the same thing with paranoid checks off
|
|
|
|
options.paranoid_checks = false;
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-10-28 20:36:02 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "bar"));
|
|
|
|
ASSERT_OK(Put(1, "foo1", "bar1"));
|
2013-10-28 20:36:02 +01:00
|
|
|
// simulate error
|
|
|
|
env_->log_write_error_.Release_Store(env_);
|
2014-02-07 23:47:16 +01:00
|
|
|
s = Put(1, "foo2", "bar2");
|
2013-10-28 20:36:02 +01:00
|
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
env_->log_write_error_.Release_Store(nullptr);
|
2014-02-07 23:47:16 +01:00
|
|
|
s = Put(1, "foo3", "bar3");
|
2013-10-28 20:36:02 +01:00
|
|
|
// the next put should NOT fail
|
|
|
|
ASSERT_TRUE(s.ok());
|
|
|
|
}
|
|
|
|
|
2012-01-25 23:56:52 +01:00
|
|
|
TEST(DBTest, FilesDeletedAfterCompaction) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "foo", "v2"));
|
|
|
|
Compact(1, "a", "z");
|
2013-08-08 00:20:41 +02:00
|
|
|
const int num_files = CountLiveFiles();
|
|
|
|
for (int i = 0; i < 10; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "foo", "v2"));
|
|
|
|
Compact(1, "a", "z");
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
ASSERT_EQ(CountLiveFiles(), num_files);
|
|
|
|
} while (ChangeCompactOptions());
|
2012-01-25 23:56:52 +01:00
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
TEST(DBTest, BloomFilter) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
env_->count_random_reads_ = true;
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
2013-11-13 07:46:51 +01:00
|
|
|
options.no_block_cache = true;
|
2013-08-08 00:20:41 +02:00
|
|
|
options.filter_policy = NewBloomFilterPolicy(10);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Populate multiple layers
|
|
|
|
const int N = 10000;
|
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), Key(i)));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
Compact(1, "a", "z");
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < N; i += 100) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, Key(i), Key(i)));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
Flush(1);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Prevent auto compactions triggered by seeks
|
|
|
|
env_->delay_sstable_sync_.Release_Store(env_);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Lookup present keys. Should rarely read from small sstable.
|
|
|
|
env_->random_read_counter_.Reset();
|
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(Key(i), Get(1, Key(i)));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
int reads = env_->random_read_counter_.Read();
|
|
|
|
fprintf(stderr, "%d present => %d reads\n", N, reads);
|
|
|
|
ASSERT_GE(reads, N);
|
|
|
|
ASSERT_LE(reads, N + 2*N/100);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Lookup present keys. Should rarely read from either sstable.
|
|
|
|
env_->random_read_counter_.Reset();
|
|
|
|
for (int i = 0; i < N; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ("NOT_FOUND", Get(1, Key(i) + ".missing"));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
reads = env_->random_read_counter_.Read();
|
|
|
|
fprintf(stderr, "%d missing => %d reads\n", N, reads);
|
|
|
|
ASSERT_LE(reads, 3*N/100);
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
env_->delay_sstable_sync_.Release_Store(nullptr);
|
|
|
|
Close();
|
|
|
|
delete options.filter_policy;
|
|
|
|
} while (ChangeCompactOptions());
|
2012-04-17 17:36:46 +02:00
|
|
|
}
|
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
TEST(DBTest, SnapshotFiles) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 100000000; // Large write buffer
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2012-09-15 02:11:35 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
Random rnd(301);
|
2012-09-24 23:01:01 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Write 8MB (80 values, each 100K)
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
2013-08-08 00:20:41 +02:00
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 80; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 100000));
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put((i < 40), Key(i), values[i]));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2012-09-24 23:01:01 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// assert that nothing makes it to disk yet.
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 0);
|
2012-09-24 23:01:01 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// get a file snapshot
|
|
|
|
uint64_t manifest_number = 0;
|
|
|
|
uint64_t manifest_size = 0;
|
|
|
|
std::vector<std::string> files;
|
|
|
|
dbfull()->DisableFileDeletions();
|
|
|
|
dbfull()->GetLiveFiles(files, &manifest_size);
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
// CURRENT, MANIFEST, *.sst files (one for each CF)
|
|
|
|
ASSERT_EQ(files.size(), 4U);
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
uint64_t number = 0;
|
|
|
|
FileType type;
|
|
|
|
|
|
|
|
// copy these files to a new snapshot directory
|
|
|
|
std::string snapdir = dbname_ + ".snapdir/";
|
|
|
|
std::string mkdir = "mkdir -p " + snapdir;
|
|
|
|
ASSERT_EQ(system(mkdir.c_str()), 0);
|
|
|
|
|
|
|
|
for (unsigned int i = 0; i < files.size(); i++) {
|
2013-11-09 00:23:46 +01:00
|
|
|
// our clients require that GetLiveFiles returns
|
|
|
|
// files with "/" as first character!
|
|
|
|
ASSERT_EQ(files[i][0], '/');
|
|
|
|
std::string src = dbname_ + files[i];
|
|
|
|
std::string dest = snapdir + files[i];
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
uint64_t size;
|
|
|
|
ASSERT_OK(env_->GetFileSize(src, &size));
|
|
|
|
|
|
|
|
// record the number and the size of the
|
|
|
|
// latest manifest file
|
|
|
|
if (ParseFileName(files[i].substr(1), &number, &type)) {
|
|
|
|
if (type == kDescriptorFile) {
|
|
|
|
if (number > manifest_number) {
|
|
|
|
manifest_number = number;
|
|
|
|
ASSERT_GE(size, manifest_size);
|
|
|
|
size = manifest_size; // copy only valid MANIFEST data
|
|
|
|
}
|
2012-09-24 23:01:01 +02:00
|
|
|
}
|
|
|
|
}
|
Refactor Recover() code
Summary:
This diff does two things:
* Rethinks how we call Recover() with read_only option. Before, we call it with pointer to memtable where we'd like to apply those changes to. This memtable is set in db_impl_readonly.cc and it's actually DBImpl::mem_. Why don't we just apply updates to mem_ right away? It seems more intuitive.
* Changes when we apply updates to manifest. Before, the process is to recover all the logs, flush it to sst files and then do one giant commit that atomically adds all recovered sst files and sets the next log number. This works good enough, but causes some small troubles for my column family approach, since I can't have one VersionEdit apply to more than single column family[1]. The change here is to commit the files recovered from logs right away. Here is the state of the world before the change:
1. Recover log 5, add new sst files to edit
2. Recover log 7, add new sst files to edit
3. Recover log 8, add new sst files to edit
4. Commit all added sst files to manifest and mark log files 5, 7 and 8 as recoverd (via SetLogNumber(9) function)
After the change, we'll do:
1. Recover log 5, commit the new sst files and set log 5 as recovered
2. Recover log 7, commit the new sst files and set log 7 as recovered
3. Recover log 8, commit the new sst files and set log 8 as recovered
The added (small) benefit is that if we fail after (2), the new recovery will only have to recover log 8. In previous case, we'll have to restart the recovery from the beginning. The bigger benefit will be to enable easier integration of multiple column families in Recovery code path.
[1] I'm happy to dicuss this decison, but I believe this is the cleanest way to go. It also makes backward compatibility much easier. We don't have a requirement of adding multiple column families atomically.
Test Plan: make check
Reviewers: dhruba, haobo, kailiu, sdong
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D15237
2014-01-22 19:45:26 +01:00
|
|
|
CopyFile(src, dest, size);
|
2012-09-24 23:01:01 +02:00
|
|
|
}
|
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// release file snapshot
|
|
|
|
dbfull()->DisableFileDeletions();
|
2012-09-15 02:11:35 +02:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// overwrite one key, this key should not appear in the snapshot
|
|
|
|
std::vector<std::string> extras;
|
|
|
|
for (unsigned int i = 0; i < 1; i++) {
|
|
|
|
extras.push_back(RandomString(&rnd, 100000));
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(0, Key(i), extras[i]));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// verify that data in the snapshot are correct
|
2014-02-07 23:47:16 +01:00
|
|
|
std::vector<ColumnFamilyDescriptor> column_families;
|
|
|
|
column_families.emplace_back("default", ColumnFamilyOptions());
|
|
|
|
column_families.emplace_back("pikachu", ColumnFamilyOptions());
|
|
|
|
std::vector<ColumnFamilyHandle*> cf_handles;
|
2013-08-08 00:20:41 +02:00
|
|
|
DB* snapdb;
|
2014-02-07 23:47:16 +01:00
|
|
|
DBOptions opts;
|
2013-08-08 00:20:41 +02:00
|
|
|
opts.create_if_missing = false;
|
2014-02-07 23:47:16 +01:00
|
|
|
Status stat =
|
|
|
|
DB::Open(opts, snapdir, column_families, &cf_handles, &snapdb);
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(stat);
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
ReadOptions roptions;
|
|
|
|
std::string val;
|
|
|
|
for (unsigned int i = 0; i < 80; i++) {
|
2014-02-07 23:47:16 +01:00
|
|
|
stat = snapdb->Get(roptions, cf_handles[i < 40], Key(i), &val);
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_EQ(values[i].compare(val), 0);
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
for (auto cfh : cf_handles) {
|
|
|
|
delete cfh;
|
|
|
|
}
|
2013-08-08 00:20:41 +02:00
|
|
|
delete snapdb;
|
|
|
|
|
|
|
|
// look at the new live files after we added an 'extra' key
|
|
|
|
// and after we took the first snapshot.
|
|
|
|
uint64_t new_manifest_number = 0;
|
|
|
|
uint64_t new_manifest_size = 0;
|
|
|
|
std::vector<std::string> newfiles;
|
|
|
|
dbfull()->DisableFileDeletions();
|
|
|
|
dbfull()->GetLiveFiles(newfiles, &new_manifest_size);
|
|
|
|
|
|
|
|
// find the new manifest file. assert that this manifest file is
|
|
|
|
// the same one as in the previous snapshot. But its size should be
|
|
|
|
// larger because we added an extra key after taking the
|
|
|
|
// previous shapshot.
|
|
|
|
for (unsigned int i = 0; i < newfiles.size(); i++) {
|
|
|
|
std::string src = dbname_ + "/" + newfiles[i];
|
|
|
|
// record the lognumber and the size of the
|
|
|
|
// latest manifest file
|
|
|
|
if (ParseFileName(newfiles[i].substr(1), &number, &type)) {
|
|
|
|
if (type == kDescriptorFile) {
|
|
|
|
if (number > new_manifest_number) {
|
|
|
|
uint64_t size;
|
|
|
|
new_manifest_number = number;
|
|
|
|
ASSERT_OK(env_->GetFileSize(src, &size));
|
|
|
|
ASSERT_GE(size, new_manifest_size);
|
|
|
|
}
|
2012-09-24 23:01:01 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_EQ(manifest_number, new_manifest_number);
|
|
|
|
ASSERT_GT(new_manifest_size, manifest_size);
|
2012-11-01 18:50:08 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// release file snapshot
|
|
|
|
dbfull()->DisableFileDeletions();
|
|
|
|
} while (ChangeCompactOptions());
|
2012-09-15 02:11:35 +02:00
|
|
|
}
|
|
|
|
|
2013-02-28 23:09:30 +01:00
|
|
|
TEST(DBTest, CompactOnFlush) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.purge_redundant_kvs_while_flush = true;
|
|
|
|
options.disable_auto_compactions = true;
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "v1");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Write two new keys
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "a", "begin");
|
|
|
|
Put(1, "z", "end");
|
|
|
|
Flush(1);
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Case1: Delete followed by a put
|
2014-02-07 23:47:16 +01:00
|
|
|
Delete(1, "foo");
|
|
|
|
Put(1, "foo", "v2");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// After the current memtable is flushed, the DEL should
|
|
|
|
// have been removed
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Case 2: Delete followed by another delete
|
2014-02-07 23:47:16 +01:00
|
|
|
Delete(1, "foo");
|
|
|
|
Delete(1, "foo");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]");
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Case 3: Put followed by a delete
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "v3");
|
|
|
|
Delete(1, "foo");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]");
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Case 4: Put followed by another Put
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "v4");
|
|
|
|
Put(1, "foo", "v5");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// clear database
|
2014-02-07 23:47:16 +01:00
|
|
|
Delete(1, "foo");
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Case 5: Put followed by snapshot followed by another Put
|
|
|
|
// Both puts should remain.
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "v6");
|
2013-08-08 00:20:41 +02:00
|
|
|
const Snapshot* snapshot = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "v7");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]");
|
2013-08-08 00:20:41 +02:00
|
|
|
db_->ReleaseSnapshot(snapshot);
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// clear database
|
2014-02-07 23:47:16 +01:00
|
|
|
Delete(1, "foo");
|
|
|
|
dbfull()->CompactRange(handles_[1], nullptr, nullptr);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
|
2013-02-28 23:09:30 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
// Case 5: snapshot followed by a put followed by another Put
|
|
|
|
// Only the last put should remain.
|
|
|
|
const Snapshot* snapshot1 = db_->GetSnapshot();
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "foo", "v8");
|
|
|
|
Put(1, "foo", "v9");
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]");
|
2013-08-08 00:20:41 +02:00
|
|
|
db_->ReleaseSnapshot(snapshot1);
|
|
|
|
} while (ChangeCompactOptions());
|
2013-02-28 23:09:30 +01:00
|
|
|
}
|
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2013-05-06 20:41:01 +02:00
|
|
|
std::vector<std::uint64_t> ListLogFiles(Env* env, const std::string& path) {
|
2012-11-26 22:56:45 +01:00
|
|
|
std::vector<std::string> files;
|
2013-05-06 20:41:01 +02:00
|
|
|
std::vector<uint64_t> log_files;
|
2012-11-26 22:56:45 +01:00
|
|
|
env->GetChildren(path, &files);
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
for (size_t i = 0; i < files.size(); ++i) {
|
|
|
|
if (ParseFileName(files[i], &number, &type)) {
|
|
|
|
if (type == kLogFile) {
|
2013-05-06 20:41:01 +02:00
|
|
|
log_files.push_back(number);
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2013-05-06 20:41:01 +02:00
|
|
|
return std::move(log_files);
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
TEST(DBTest, WALArchivalTtl) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.WAL_ttl_seconds = 1000;
|
|
|
|
DestroyAndReopen(&options);
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
// TEST : Create DB with a ttl and no size limit.
|
2013-08-08 00:20:41 +02:00
|
|
|
// Put some keys. Count the log files present in the DB just after insert.
|
|
|
|
// Re-open db. Causes deletion/archival to take place.
|
|
|
|
// Assert that the files moved under "/archive".
|
2013-11-07 03:46:28 +01:00
|
|
|
// Reopen db with small ttl.
|
|
|
|
// Assert that archive was removed.
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
std::string archiveDir = ArchivalDirectory(dbname_);
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
for (int i = 0; i < 10; ++i) {
|
|
|
|
for (int j = 0; j < 10; ++j) {
|
2013-11-07 03:46:28 +01:00
|
|
|
ASSERT_OK(Put(Key(10 * i + j), DummyString(1024)));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
std::vector<uint64_t> log_files = ListLogFiles(env_, dbname_);
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
options.create_if_missing = false;
|
|
|
|
Reopen(&options);
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
std::vector<uint64_t> logs = ListLogFiles(env_, archiveDir);
|
|
|
|
std::set<uint64_t> archivedFiles(logs.begin(), logs.end());
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
for (auto& log : log_files) {
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_TRUE(archivedFiles.find(log) != archivedFiles.end());
|
|
|
|
}
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
std::vector<uint64_t> log_files = ListLogFiles(env_, archiveDir);
|
|
|
|
ASSERT_TRUE(log_files.size() > 0);
|
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
options.WAL_ttl_seconds = 1;
|
2013-11-07 03:46:28 +01:00
|
|
|
env_->SleepForMicroseconds(2 * 1000 * 1000);
|
2013-08-08 00:20:41 +02:00
|
|
|
Reopen(&options);
|
2012-11-26 22:56:45 +01:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
log_files = ListLogFiles(env_, archiveDir);
|
|
|
|
ASSERT_TRUE(log_files.empty());
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2012-11-26 22:56:45 +01:00
|
|
|
}
|
2012-09-27 10:05:38 +02:00
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2013-11-07 03:46:28 +01:00
|
|
|
uint64_t GetLogDirSize(std::string dir_path, SpecialEnv* env) {
|
|
|
|
uint64_t dir_size = 0;
|
|
|
|
std::vector<std::string> files;
|
|
|
|
env->GetChildren(dir_path, &files);
|
|
|
|
for (auto& f : files) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
if (ParseFileName(f, &number, &type) && type == kLogFile) {
|
|
|
|
std::string const file_path = dir_path + "/" + f;
|
|
|
|
uint64_t file_size;
|
|
|
|
env->GetFileSize(file_path, &file_size);
|
|
|
|
dir_size += file_size;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return dir_size;
|
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2013-11-07 03:46:28 +01:00
|
|
|
|
|
|
|
TEST(DBTest, WALArchivalSizeLimit) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
2013-11-07 03:46:28 +01:00
|
|
|
options.WAL_ttl_seconds = 0;
|
|
|
|
options.WAL_size_limit_MB = 1000;
|
|
|
|
|
|
|
|
// TEST : Create DB with huge size limit and no ttl.
|
|
|
|
// Put some keys. Count the archived log files present in the DB
|
|
|
|
// just after insert. Assert that there are many enough.
|
|
|
|
// Change size limit. Re-open db.
|
|
|
|
// Assert that archive is not greater than WAL_size_limit_MB.
|
|
|
|
// Set ttl and time_to_check_ to small values. Re-open db.
|
|
|
|
// Assert that there are no archived logs left.
|
2013-05-06 20:41:01 +02:00
|
|
|
|
2013-11-07 03:46:28 +01:00
|
|
|
DestroyAndReopen(&options);
|
|
|
|
for (int i = 0; i < 128 * 128; ++i) {
|
|
|
|
ASSERT_OK(Put(Key(i), DummyString(1024)));
|
|
|
|
}
|
2013-08-08 00:20:41 +02:00
|
|
|
Reopen(&options);
|
2013-11-07 03:46:28 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
std::string archive_dir = ArchivalDirectory(dbname_);
|
|
|
|
std::vector<std::uint64_t> log_files = ListLogFiles(env_, archive_dir);
|
2013-11-07 03:46:28 +01:00
|
|
|
ASSERT_TRUE(log_files.size() > 2);
|
|
|
|
|
|
|
|
options.WAL_size_limit_MB = 8;
|
|
|
|
Reopen(&options);
|
|
|
|
dbfull()->TEST_PurgeObsoleteteWAL();
|
|
|
|
|
|
|
|
uint64_t archive_size = GetLogDirSize(archive_dir, env_);
|
|
|
|
ASSERT_TRUE(archive_size <= options.WAL_size_limit_MB * 1024 * 1024);
|
|
|
|
|
|
|
|
options.WAL_ttl_seconds = 1;
|
|
|
|
dbfull()->TEST_SetDefaultTimeToCheck(1);
|
2013-08-08 00:20:41 +02:00
|
|
|
env_->SleepForMicroseconds(2 * 1000 * 1000);
|
2013-11-07 03:46:28 +01:00
|
|
|
Reopen(&options);
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->TEST_PurgeObsoleteteWAL();
|
2013-11-07 03:46:28 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
log_files = ListLogFiles(env_, archive_dir);
|
|
|
|
ASSERT_TRUE(log_files.empty());
|
|
|
|
} while (ChangeCompactOptions());
|
2013-05-06 20:41:01 +02:00
|
|
|
}
|
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2013-10-25 04:09:02 +02:00
|
|
|
SequenceNumber ReadRecords(
|
|
|
|
std::unique_ptr<TransactionLogIterator>& iter,
|
|
|
|
int& count) {
|
|
|
|
count = 0;
|
2013-03-21 23:12:35 +01:00
|
|
|
SequenceNumber lastSequence = 0;
|
2013-10-25 04:09:02 +02:00
|
|
|
BatchResult res;
|
2013-03-21 23:12:35 +01:00
|
|
|
while (iter->Valid()) {
|
2013-10-25 04:09:02 +02:00
|
|
|
res = iter->GetBatch();
|
2013-03-21 23:12:35 +01:00
|
|
|
ASSERT_TRUE(res.sequence > lastSequence);
|
2013-10-25 04:09:02 +02:00
|
|
|
++count;
|
2013-03-21 23:12:35 +01:00
|
|
|
lastSequence = res.sequence;
|
2013-10-04 19:21:03 +02:00
|
|
|
ASSERT_OK(iter->status());
|
2013-03-21 23:12:35 +01:00
|
|
|
iter->Next();
|
|
|
|
}
|
2013-10-25 04:09:02 +02:00
|
|
|
return res.sequence;
|
|
|
|
}
|
|
|
|
|
|
|
|
void ExpectRecords(
|
|
|
|
const int expected_no_records,
|
|
|
|
std::unique_ptr<TransactionLogIterator>& iter) {
|
|
|
|
int num_records;
|
|
|
|
ReadRecords(iter, num_records);
|
|
|
|
ASSERT_EQ(num_records, expected_no_records);
|
2013-03-21 23:12:35 +01:00
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2013-03-21 23:12:35 +01:00
|
|
|
|
2012-11-30 02:28:37 +01:00
|
|
|
TEST(DBTest, TransactionLogIterator) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
|
|
|
Put(0, "key1", DummyString(1024));
|
|
|
|
Put(1, "key2", DummyString(1024));
|
|
|
|
Put(1, "key2", DummyString(1024));
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3U);
|
|
|
|
{
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ExpectRecords(3, iter);
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
env_->SleepForMicroseconds(2 * 1000 * 1000);
|
|
|
|
{
|
|
|
|
Put(0, "key4", DummyString(1024));
|
|
|
|
Put(1, "key5", DummyString(1024));
|
|
|
|
Put(0, "key6", DummyString(1024));
|
2013-08-08 00:20:41 +02:00
|
|
|
}
|
|
|
|
{
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ExpectRecords(6, iter);
|
|
|
|
}
|
|
|
|
} while (ChangeCompactOptions());
|
2012-11-30 02:28:37 +01:00
|
|
|
}
|
|
|
|
|
2014-04-21 20:08:30 +02:00
|
|
|
#ifndef NDEBUG // sync point is not included with DNDEBUG build
|
2014-03-24 05:49:14 +01:00
|
|
|
TEST(DBTest, TransactionLogIteratorRace) {
|
2014-05-13 02:50:21 +02:00
|
|
|
static const int LOG_ITERATOR_RACE_TEST_COUNT = 2;
|
|
|
|
static const char* sync_points[LOG_ITERATOR_RACE_TEST_COUNT][4] =
|
|
|
|
{ { "DBImpl::GetSortedWalFiles:1", "DBImpl::PurgeObsoleteFiles:1",
|
|
|
|
"DBImpl::PurgeObsoleteFiles:2", "DBImpl::GetSortedWalFiles:2" },
|
|
|
|
{ "DBImpl::GetSortedWalsOfType:1", "DBImpl::PurgeObsoleteFiles:1",
|
|
|
|
"DBImpl::PurgeObsoleteFiles:2", "DBImpl::GetSortedWalsOfType:2" }};
|
|
|
|
for (int test = 0; test < LOG_ITERATOR_RACE_TEST_COUNT; ++test) {
|
|
|
|
// Setup sync point dependency to reproduce the race condition of
|
|
|
|
// a log file moved to archived dir, in the middle of GetSortedWalFiles
|
|
|
|
rocksdb::SyncPoint::GetInstance()->LoadDependency(
|
|
|
|
{ { sync_points[test][0], sync_points[test][1] },
|
|
|
|
{ sync_points[test][2], sync_points[test][3] },
|
|
|
|
});
|
|
|
|
|
|
|
|
do {
|
|
|
|
rocksdb::SyncPoint::GetInstance()->ClearTrace();
|
|
|
|
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
Put("key1", DummyString(1024));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
Put("key2", DummyString(1024));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
Put("key3", DummyString(1024));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
Put("key4", DummyString(1024));
|
|
|
|
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4U);
|
|
|
|
|
|
|
|
{
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ExpectRecords(4, iter);
|
|
|
|
}
|
2014-03-24 05:49:14 +01:00
|
|
|
|
2014-05-13 02:50:21 +02:00
|
|
|
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
|
|
|
// trigger async flush, and log move. Well, log move will
|
|
|
|
// wait until the GetSortedWalFiles:1 to reproduce the race
|
|
|
|
// condition
|
|
|
|
FlushOptions flush_options;
|
|
|
|
flush_options.wait = false;
|
|
|
|
dbfull()->Flush(flush_options);
|
|
|
|
|
|
|
|
// "key5" would be written in a new memtable and log
|
|
|
|
Put("key5", DummyString(1024));
|
|
|
|
{
|
|
|
|
// this iter would miss "key4" if not fixed
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ExpectRecords(5, iter);
|
|
|
|
}
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
2014-03-24 05:49:14 +01:00
|
|
|
}
|
2014-04-21 20:08:30 +02:00
|
|
|
#endif
|
2014-03-24 05:49:14 +01:00
|
|
|
|
2013-03-18 22:50:59 +01:00
|
|
|
TEST(DBTest, TransactionLogIteratorMoveOverZeroFiles) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-08 00:20:41 +02:00
|
|
|
// Do a plain Reopen.
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "key1", DummyString(1024));
|
2013-08-08 00:20:41 +02:00
|
|
|
// Two reopens should create a zero record WAL file.
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2013-03-18 22:50:59 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
Put(1, "key2", DummyString(1024));
|
2013-03-21 23:12:35 +01:00
|
|
|
|
2013-08-08 00:20:41 +02:00
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ExpectRecords(2, iter);
|
|
|
|
} while (ChangeCompactOptions());
|
2013-03-21 23:12:35 +01:00
|
|
|
}
|
2013-03-28 21:13:35 +01:00
|
|
|
|
|
|
|
TEST(DBTest, TransactionLogIteratorStallAtLastRecord) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
Put("key1", DummyString(1024));
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
Put("key2", DummyString(1024));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
} while (ChangeCompactOptions());
|
2013-03-28 21:13:35 +01:00
|
|
|
}
|
|
|
|
|
2013-03-28 21:19:07 +01:00
|
|
|
TEST(DBTest, TransactionLogIteratorJustEmptyFile) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
unique_ptr<TransactionLogIterator> iter;
|
|
|
|
Status status = dbfull()->GetUpdatesSince(0, &iter);
|
2013-10-14 00:28:24 +02:00
|
|
|
// Check that an empty iterator is returned
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2013-03-28 21:19:07 +01:00
|
|
|
}
|
2013-04-03 02:18:27 +02:00
|
|
|
|
|
|
|
TEST(DBTest, TransactionLogIteratorCheckAfterRestart) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
Put("key1", DummyString(1024));
|
|
|
|
Put("key2", DummyString(1023));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
Reopen(&options);
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
ExpectRecords(2, iter);
|
|
|
|
} while (ChangeCompactOptions());
|
2013-04-03 02:18:27 +02:00
|
|
|
}
|
|
|
|
|
2013-10-25 04:09:02 +02:00
|
|
|
TEST(DBTest, TransactionLogIteratorCorruptedLog) {
|
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
for (int i = 0; i < 1024; i++) {
|
|
|
|
Put("key"+std::to_string(i), DummyString(10));
|
|
|
|
}
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
// Corrupt this log to create a gap
|
|
|
|
rocksdb::VectorLogPtr wal_files;
|
|
|
|
ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
|
|
|
|
const auto logfilePath = dbname_ + "/" + wal_files.front()->PathName();
|
|
|
|
ASSERT_EQ(
|
|
|
|
0,
|
|
|
|
truncate(logfilePath.c_str(), wal_files.front()->SizeFileBytes() / 2));
|
|
|
|
// Insert a new entry to a new log file
|
|
|
|
Put("key1025", DummyString(10));
|
|
|
|
// Try to read from the beginning. Should stop before the gap and read less
|
|
|
|
// than 1025 entries
|
|
|
|
auto iter = OpenTransactionLogIter(0);
|
|
|
|
int count;
|
|
|
|
int last_sequence_read = ReadRecords(iter, count);
|
|
|
|
ASSERT_LT(last_sequence_read, 1025);
|
|
|
|
// Try to read past the gap, should be able to seek to key1025
|
|
|
|
auto iter2 = OpenTransactionLogIter(last_sequence_read + 1);
|
|
|
|
ExpectRecords(1, iter2);
|
|
|
|
} while (ChangeCompactOptions());
|
|
|
|
}
|
|
|
|
|
2013-04-09 01:28:09 +02:00
|
|
|
TEST(DBTest, TransactionLogIteratorBatchOperations) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-08 00:20:41 +02:00
|
|
|
WriteBatch batch;
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Put(handles_[1], "key1", DummyString(1024));
|
|
|
|
batch.Put(handles_[0], "key2", DummyString(1024));
|
|
|
|
batch.Put(handles_[1], "key3", DummyString(1024));
|
|
|
|
batch.Delete(handles_[0], "key2");
|
2013-08-08 00:20:41 +02:00
|
|
|
dbfull()->Write(WriteOptions(), &batch);
|
2014-02-07 23:47:16 +01:00
|
|
|
Flush(1);
|
|
|
|
Flush(0);
|
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
|
|
|
Put(1, "key4", DummyString(1024));
|
2013-08-08 00:20:41 +02:00
|
|
|
auto iter = OpenTransactionLogIter(3);
|
2013-10-14 00:28:24 +02:00
|
|
|
ExpectRecords(2, iter);
|
2013-08-08 00:20:41 +02:00
|
|
|
} while (ChangeCompactOptions());
|
2013-04-09 01:28:09 +02:00
|
|
|
}
|
|
|
|
|
2013-08-15 01:32:46 +02:00
|
|
|
TEST(DBTest, TransactionLogIteratorBlobs) {
|
|
|
|
Options options = OptionsForLogIterTest();
|
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2013-08-15 01:32:46 +02:00
|
|
|
{
|
|
|
|
WriteBatch batch;
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Put(handles_[1], "key1", DummyString(1024));
|
|
|
|
batch.Put(handles_[0], "key2", DummyString(1024));
|
2013-08-15 01:32:46 +02:00
|
|
|
batch.PutLogData(Slice("blob1"));
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Put(handles_[1], "key3", DummyString(1024));
|
2013-08-15 01:32:46 +02:00
|
|
|
batch.PutLogData(Slice("blob2"));
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Delete(handles_[0], "key2");
|
2013-08-15 01:32:46 +02:00
|
|
|
dbfull()->Write(WriteOptions(), &batch);
|
2014-02-07 23:47:16 +01:00
|
|
|
ReopenWithColumnFamilies({"default", "pikachu"}, &options);
|
2013-08-15 01:32:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
auto res = OpenTransactionLogIter(0)->GetBatch();
|
|
|
|
struct Handler : public WriteBatch::Handler {
|
|
|
|
std::string seen;
|
2014-02-26 02:30:54 +01:00
|
|
|
virtual Status PutCF(uint32_t cf, const Slice& key, const Slice& value) {
|
2014-02-07 23:47:16 +01:00
|
|
|
seen += "Put(" + std::to_string(cf) + ", " + key.ToString() + ", " +
|
|
|
|
std::to_string(value.size()) + ")";
|
2014-02-26 02:30:54 +01:00
|
|
|
return Status::OK();
|
2013-08-15 01:32:46 +02:00
|
|
|
}
|
2014-02-26 02:30:54 +01:00
|
|
|
virtual Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) {
|
2014-02-07 23:47:16 +01:00
|
|
|
seen += "Merge(" + std::to_string(cf) + ", " + key.ToString() + ", " +
|
|
|
|
std::to_string(value.size()) + ")";
|
2014-02-26 02:30:54 +01:00
|
|
|
return Status::OK();
|
2013-08-15 01:32:46 +02:00
|
|
|
}
|
|
|
|
virtual void LogData(const Slice& blob) {
|
|
|
|
seen += "LogData(" + blob.ToString() + ")";
|
|
|
|
}
|
2014-02-26 02:30:54 +01:00
|
|
|
virtual Status DeleteCF(uint32_t cf, const Slice& key) {
|
2014-02-07 23:47:16 +01:00
|
|
|
seen += "Delete(" + std::to_string(cf) + ", " + key.ToString() + ")";
|
2014-02-26 02:30:54 +01:00
|
|
|
return Status::OK();
|
2013-08-15 01:32:46 +02:00
|
|
|
}
|
|
|
|
} handler;
|
|
|
|
res.writeBatchPtr->Iterate(&handler);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_EQ(
|
|
|
|
"Put(1, key1, 1024)"
|
|
|
|
"Put(0, key2, 1024)"
|
|
|
|
"LogData(blob1)"
|
|
|
|
"Put(1, key3, 1024)"
|
|
|
|
"LogData(blob2)"
|
|
|
|
"Delete(0, key2)",
|
|
|
|
handler.seen);
|
2013-08-15 01:32:46 +02:00
|
|
|
}
|
|
|
|
|
2014-04-29 19:27:58 +02:00
|
|
|
TEST(DBTest, ReadFirstRecordCache) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
std::string path = dbname_ + "/000001.log";
|
|
|
|
unique_ptr<WritableFile> file;
|
|
|
|
ASSERT_OK(env_->NewWritableFile(path, &file, EnvOptions()));
|
|
|
|
|
|
|
|
SequenceNumber s;
|
|
|
|
ASSERT_OK(dbfull()->TEST_ReadFirstLine(path, &s));
|
2014-04-29 21:47:48 +02:00
|
|
|
ASSERT_EQ(s, 0U);
|
2014-04-29 19:27:58 +02:00
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->TEST_ReadFirstRecord(kAliveLogFile, 1, &s));
|
2014-04-29 21:47:48 +02:00
|
|
|
ASSERT_EQ(s, 0U);
|
2014-04-29 19:27:58 +02:00
|
|
|
|
|
|
|
log::Writer writer(std::move(file));
|
|
|
|
WriteBatch batch;
|
|
|
|
batch.Put("foo", "bar");
|
|
|
|
WriteBatchInternal::SetSequence(&batch, 10);
|
|
|
|
writer.AddRecord(WriteBatchInternal::Contents(&batch));
|
|
|
|
|
|
|
|
env_->count_sequential_reads_ = true;
|
|
|
|
// sequential_read_counter_ sanity test
|
|
|
|
ASSERT_EQ(env_->sequential_read_counter_.Read(), 0);
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->TEST_ReadFirstRecord(kAliveLogFile, 1, &s));
|
2014-04-29 21:47:48 +02:00
|
|
|
ASSERT_EQ(s, 10U);
|
2014-04-29 19:27:58 +02:00
|
|
|
// did a read
|
|
|
|
ASSERT_EQ(env_->sequential_read_counter_.Read(), 1);
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->TEST_ReadFirstRecord(kAliveLogFile, 1, &s));
|
2014-04-29 21:47:48 +02:00
|
|
|
ASSERT_EQ(s, 10U);
|
2014-04-29 19:27:58 +02:00
|
|
|
// no new reads since the value is cached
|
|
|
|
ASSERT_EQ(env_->sequential_read_counter_.Read(), 1);
|
|
|
|
}
|
|
|
|
|
2011-05-28 02:53:58 +02:00
|
|
|
// Multi-threaded test:
|
|
|
|
namespace {
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
static const int kColumnFamilies = 10;
|
|
|
|
static const int kNumThreads = 10;
|
2011-05-28 02:53:58 +02:00
|
|
|
static const int kTestSeconds = 10;
|
|
|
|
static const int kNumKeys = 1000;
|
|
|
|
|
|
|
|
struct MTState {
|
|
|
|
DBTest* test;
|
|
|
|
port::AtomicPointer stop;
|
|
|
|
port::AtomicPointer counter[kNumThreads];
|
|
|
|
port::AtomicPointer thread_done[kNumThreads];
|
|
|
|
};
|
|
|
|
|
|
|
|
struct MTThread {
|
|
|
|
MTState* state;
|
|
|
|
int id;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void MTThreadBody(void* arg) {
|
|
|
|
MTThread* t = reinterpret_cast<MTThread*>(arg);
|
2012-01-25 23:56:52 +01:00
|
|
|
int id = t->id;
|
2011-05-28 02:53:58 +02:00
|
|
|
DB* db = t->state->test->db_;
|
|
|
|
uintptr_t counter = 0;
|
2012-01-25 23:56:52 +01:00
|
|
|
fprintf(stderr, "... starting thread %d\n", id);
|
|
|
|
Random rnd(1000 + id);
|
2011-05-28 02:53:58 +02:00
|
|
|
char valbuf[1500];
|
2013-03-01 03:04:58 +01:00
|
|
|
while (t->state->stop.Acquire_Load() == nullptr) {
|
2012-01-25 23:56:52 +01:00
|
|
|
t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter));
|
2011-05-28 02:53:58 +02:00
|
|
|
|
|
|
|
int key = rnd.Uniform(kNumKeys);
|
|
|
|
char keybuf[20];
|
|
|
|
snprintf(keybuf, sizeof(keybuf), "%016d", key);
|
|
|
|
|
|
|
|
if (rnd.OneIn(2)) {
|
2014-02-07 23:47:16 +01:00
|
|
|
// Write values of the form <key, my id, counter, cf, unique_id>.
|
|
|
|
// into each of the CFs
|
2011-05-28 02:53:58 +02:00
|
|
|
// We add some padding for force compactions.
|
2014-02-07 23:47:16 +01:00
|
|
|
int unique_id = rnd.Uniform(1000000);
|
|
|
|
WriteBatch batch;
|
|
|
|
for (int cf = 0; cf < kColumnFamilies; ++cf) {
|
|
|
|
snprintf(valbuf, sizeof(valbuf), "%d.%d.%d.%d.%-1000d", key, id,
|
|
|
|
static_cast<int>(counter), cf, unique_id);
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Put(t->state->test->handles_[cf], Slice(keybuf), Slice(valbuf));
|
2014-02-07 23:47:16 +01:00
|
|
|
}
|
|
|
|
ASSERT_OK(db->Write(WriteOptions(), &batch));
|
2011-05-28 02:53:58 +02:00
|
|
|
} else {
|
2014-02-07 23:47:16 +01:00
|
|
|
// Read a value and verify that it matches the pattern written above
|
|
|
|
// and that writes to all column families were atomic (unique_id is the
|
|
|
|
// same)
|
|
|
|
std::vector<Slice> keys(kColumnFamilies, Slice(keybuf));
|
|
|
|
std::vector<std::string> values;
|
|
|
|
std::vector<Status> statuses =
|
|
|
|
db->MultiGet(ReadOptions(), t->state->test->handles_, keys, &values);
|
|
|
|
Status s = statuses[0];
|
|
|
|
// all statuses have to be the same
|
|
|
|
for (size_t i = 1; i < statuses.size(); ++i) {
|
|
|
|
// they are either both ok or both not-found
|
|
|
|
ASSERT_TRUE((s.ok() && statuses[i].ok()) ||
|
|
|
|
(s.IsNotFound() && statuses[i].IsNotFound()));
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
if (s.IsNotFound()) {
|
|
|
|
// Key has not yet been written
|
|
|
|
} else {
|
|
|
|
// Check that the writer thread counter is >= the counter in the value
|
|
|
|
ASSERT_OK(s);
|
2014-02-07 23:47:16 +01:00
|
|
|
int unique_id = -1;
|
|
|
|
for (int i = 0; i < kColumnFamilies; ++i) {
|
|
|
|
int k, w, c, cf, u;
|
|
|
|
ASSERT_EQ(5, sscanf(values[i].c_str(), "%d.%d.%d.%d.%d", &k, &w,
|
|
|
|
&c, &cf, &u))
|
|
|
|
<< values[i];
|
|
|
|
ASSERT_EQ(k, key);
|
|
|
|
ASSERT_GE(w, 0);
|
|
|
|
ASSERT_LT(w, kNumThreads);
|
|
|
|
ASSERT_LE((unsigned int)c, reinterpret_cast<uintptr_t>(
|
|
|
|
t->state->counter[w].Acquire_Load()));
|
|
|
|
ASSERT_EQ(cf, i);
|
|
|
|
if (i == 0) {
|
|
|
|
unique_id = u;
|
|
|
|
} else {
|
|
|
|
// this checks that updates across column families happened
|
|
|
|
// atomically -- all unique ids are the same
|
|
|
|
ASSERT_EQ(u, unique_id);
|
|
|
|
}
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
counter++;
|
|
|
|
}
|
2012-01-25 23:56:52 +01:00
|
|
|
t->state->thread_done[id].Release_Store(t);
|
|
|
|
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
} // namespace
|
2011-05-28 02:53:58 +02:00
|
|
|
|
|
|
|
TEST(DBTest, MultiThreaded) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
std::vector<std::string> cfs;
|
|
|
|
for (int i = 1; i < kColumnFamilies; ++i) {
|
|
|
|
cfs.push_back(std::to_string(i));
|
|
|
|
}
|
|
|
|
CreateAndReopenWithCF(cfs);
|
2012-04-17 17:36:46 +02:00
|
|
|
// Initialize state
|
|
|
|
MTState mt;
|
|
|
|
mt.test = this;
|
|
|
|
mt.stop.Release_Store(0);
|
|
|
|
for (int id = 0; id < kNumThreads; id++) {
|
|
|
|
mt.counter[id].Release_Store(0);
|
|
|
|
mt.thread_done[id].Release_Store(0);
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Start threads
|
|
|
|
MTThread thread[kNumThreads];
|
|
|
|
for (int id = 0; id < kNumThreads; id++) {
|
|
|
|
thread[id].state = &mt;
|
|
|
|
thread[id].id = id;
|
|
|
|
env_->StartThread(MTThreadBody, &thread[id]);
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Let them run for a while
|
|
|
|
env_->SleepForMicroseconds(kTestSeconds * 1000000);
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Stop the threads and wait for them to finish
|
|
|
|
mt.stop.Release_Store(&mt);
|
|
|
|
for (int id = 0; id < kNumThreads; id++) {
|
2013-03-01 03:04:58 +01:00
|
|
|
while (mt.thread_done[id].Acquire_Load() == nullptr) {
|
2012-04-17 17:36:46 +02:00
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// skip as HashCuckooRep does not support snapshot
|
|
|
|
} while (ChangeOptions(kSkipHashCuckoo));
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
|
|
|
|
2014-01-14 23:49:31 +01:00
|
|
|
// Group commit test:
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
static const int kGCNumThreads = 4;
|
|
|
|
static const int kGCNumKeys = 1000;
|
|
|
|
|
|
|
|
struct GCThread {
|
|
|
|
DB* db;
|
|
|
|
int id;
|
|
|
|
std::atomic<bool> done;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void GCThreadBody(void* arg) {
|
|
|
|
GCThread* t = reinterpret_cast<GCThread*>(arg);
|
|
|
|
int id = t->id;
|
|
|
|
DB* db = t->db;
|
|
|
|
WriteOptions wo;
|
|
|
|
|
|
|
|
for (int i = 0; i < kGCNumKeys; ++i) {
|
|
|
|
std::string kv(std::to_string(i + id * kGCNumKeys));
|
|
|
|
ASSERT_OK(db->Put(wo, kv, kv));
|
|
|
|
}
|
|
|
|
t->done = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace
|
|
|
|
|
|
|
|
TEST(DBTest, GroupCommitTest) {
|
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.statistics = rocksdb::CreateDBStatistics();
|
|
|
|
Reopen(&options);
|
|
|
|
|
2014-01-14 23:49:31 +01:00
|
|
|
// Start threads
|
|
|
|
GCThread thread[kGCNumThreads];
|
|
|
|
for (int id = 0; id < kGCNumThreads; id++) {
|
|
|
|
thread[id].id = id;
|
|
|
|
thread[id].db = db_;
|
|
|
|
thread[id].done = false;
|
|
|
|
env_->StartThread(GCThreadBody, &thread[id]);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int id = 0; id < kGCNumThreads; id++) {
|
|
|
|
while (thread[id].done == false) {
|
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
|
|
|
}
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_GT(TestGetTickerCount(options, WRITE_DONE_BY_OTHER), 0);
|
2014-01-14 23:49:31 +01:00
|
|
|
|
|
|
|
std::vector<std::string> expected_db;
|
|
|
|
for (int i = 0; i < kGCNumThreads * kGCNumKeys; ++i) {
|
|
|
|
expected_db.push_back(std::to_string(i));
|
|
|
|
}
|
|
|
|
sort(expected_db.begin(), expected_db.end());
|
|
|
|
|
|
|
|
Iterator* itr = db_->NewIterator(ReadOptions());
|
|
|
|
itr->SeekToFirst();
|
|
|
|
for (auto x : expected_db) {
|
|
|
|
ASSERT_TRUE(itr->Valid());
|
|
|
|
ASSERT_EQ(itr->key().ToString(), x);
|
|
|
|
ASSERT_EQ(itr->value().ToString(), x);
|
|
|
|
itr->Next();
|
|
|
|
}
|
|
|
|
ASSERT_TRUE(!itr->Valid());
|
2014-01-15 00:41:30 +01:00
|
|
|
delete itr;
|
2014-01-14 23:49:31 +01:00
|
|
|
|
2014-04-25 21:21:34 +02:00
|
|
|
} while (ChangeOptions(kSkipNoSeekToLast));
|
2014-01-14 23:49:31 +01:00
|
|
|
}
|
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
namespace {
|
|
|
|
typedef std::map<std::string, std::string> KVMap;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
class ModelDB: public DB {
|
|
|
|
public:
|
2011-05-21 04:17:43 +02:00
|
|
|
class ModelSnapshot : public Snapshot {
|
|
|
|
public:
|
|
|
|
KVMap map_;
|
|
|
|
};
|
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
explicit ModelDB(const Options& options) : options_(options) {}
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::Put;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual Status Put(const WriteOptions& o, ColumnFamilyHandle* cf,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Slice& k, const Slice& v) {
|
2014-02-11 02:04:44 +01:00
|
|
|
WriteBatch batch;
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Put(cf, k, v);
|
2014-02-11 02:04:44 +01:00
|
|
|
return Write(o, &batch);
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
using DB::Merge;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual Status Merge(const WriteOptions& o, ColumnFamilyHandle* cf,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Slice& k, const Slice& v) {
|
2014-02-11 02:04:44 +01:00
|
|
|
WriteBatch batch;
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Merge(cf, k, v);
|
2014-02-11 02:04:44 +01:00
|
|
|
return Write(o, &batch);
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
using DB::Delete;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual Status Delete(const WriteOptions& o, ColumnFamilyHandle* cf,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Slice& key) {
|
2014-02-11 02:04:44 +01:00
|
|
|
WriteBatch batch;
|
2014-03-14 21:40:06 +01:00
|
|
|
batch.Delete(cf, key);
|
2014-02-11 02:04:44 +01:00
|
|
|
return Write(o, &batch);
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
}
|
|
|
|
using DB::Get;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual Status Get(const ReadOptions& options, ColumnFamilyHandle* cf,
|
2011-03-18 23:37:00 +01:00
|
|
|
const Slice& key, std::string* value) {
|
2013-06-05 20:22:38 +02:00
|
|
|
return Status::NotSupported(key);
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::MultiGet;
|
|
|
|
virtual std::vector<Status> MultiGet(
|
|
|
|
const ReadOptions& options,
|
2014-02-11 02:04:44 +01:00
|
|
|
const std::vector<ColumnFamilyHandle*>& column_family,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const std::vector<Slice>& keys, std::vector<std::string>* values) {
|
2013-06-05 20:22:38 +02:00
|
|
|
std::vector<Status> s(keys.size(),
|
|
|
|
Status::NotSupported("Not implemented."));
|
|
|
|
return s;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2014-02-15 01:46:03 +01:00
|
|
|
|
2014-02-15 02:02:10 +01:00
|
|
|
using DB::GetPropertiesOfAllTables;
|
|
|
|
virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
|
|
|
|
TablePropertiesCollection* props) {
|
2014-02-14 01:28:21 +01:00
|
|
|
return Status();
|
|
|
|
}
|
2014-02-15 01:46:03 +01:00
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::KeyMayExist;
|
2013-07-26 21:57:01 +02:00
|
|
|
virtual bool KeyMayExist(const ReadOptions& options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family, const Slice& key,
|
|
|
|
std::string* value, bool* value_found = nullptr) {
|
2013-07-26 21:57:01 +02:00
|
|
|
if (value_found != nullptr) {
|
|
|
|
*value_found = false;
|
|
|
|
}
|
2013-07-06 03:49:18 +02:00
|
|
|
return true; // Not Supported directly
|
|
|
|
}
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::NewIterator;
|
|
|
|
virtual Iterator* NewIterator(const ReadOptions& options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family) {
|
2013-03-01 03:04:58 +01:00
|
|
|
if (options.snapshot == nullptr) {
|
2011-03-18 23:37:00 +01:00
|
|
|
KVMap* saved = new KVMap;
|
|
|
|
*saved = map_;
|
|
|
|
return new ModelIter(saved, true);
|
|
|
|
} else {
|
|
|
|
const KVMap* snapshot_state =
|
2011-05-21 04:17:43 +02:00
|
|
|
&(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
|
2011-03-18 23:37:00 +01:00
|
|
|
return new ModelIter(snapshot_state, false);
|
|
|
|
}
|
|
|
|
}
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
virtual Status NewIterators(
|
|
|
|
const ReadOptions& options,
|
2014-02-11 02:04:44 +01:00
|
|
|
const std::vector<ColumnFamilyHandle*>& column_family,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
std::vector<Iterator*>* iterators) {
|
|
|
|
return Status::NotSupported("Not supported yet");
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
virtual const Snapshot* GetSnapshot() {
|
2011-05-21 04:17:43 +02:00
|
|
|
ModelSnapshot* snapshot = new ModelSnapshot;
|
|
|
|
snapshot->map_ = map_;
|
|
|
|
return snapshot;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual void ReleaseSnapshot(const Snapshot* snapshot) {
|
2011-05-21 04:17:43 +02:00
|
|
|
delete reinterpret_cast<const ModelSnapshot*>(snapshot);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
|
2011-05-21 04:17:43 +02:00
|
|
|
class Handler : public WriteBatch::Handler {
|
|
|
|
public:
|
|
|
|
KVMap* map_;
|
|
|
|
virtual void Put(const Slice& key, const Slice& value) {
|
|
|
|
(*map_)[key.ToString()] = value.ToString();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-03-21 23:59:47 +01:00
|
|
|
virtual void Merge(const Slice& key, const Slice& value) {
|
|
|
|
// ignore merge for now
|
|
|
|
//(*map_)[key.ToString()] = value.ToString();
|
|
|
|
}
|
2011-05-21 04:17:43 +02:00
|
|
|
virtual void Delete(const Slice& key) {
|
|
|
|
map_->erase(key.ToString());
|
|
|
|
}
|
|
|
|
};
|
|
|
|
Handler handler;
|
|
|
|
handler.map_ = &map_;
|
|
|
|
return batch->Iterate(&handler);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::GetProperty;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual bool GetProperty(ColumnFamilyHandle* column_family,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Slice& property, std::string* value) {
|
2011-03-18 23:37:00 +01:00
|
|
|
return false;
|
|
|
|
}
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::GetApproximateSizes;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
const Range* range, int n, uint64_t* sizes) {
|
2011-03-18 23:37:00 +01:00
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
sizes[i] = 0;
|
|
|
|
}
|
|
|
|
}
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::CompactRange;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual Status CompactRange(ColumnFamilyHandle* column_family,
|
2014-01-24 18:27:29 +01:00
|
|
|
const Slice* start, const Slice* end,
|
2014-01-22 21:46:24 +01:00
|
|
|
bool reduce_level, int target_level) {
|
|
|
|
return Status::NotSupported("Not supported operation.");
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::NumberLevels;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual int NumberLevels(ColumnFamilyHandle* column_family) { return 1; }
|
2012-06-23 04:30:03 +02:00
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::MaxMemCompactionLevel;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
return 1;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::Level0StopWriteTrigger;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual int Level0StopWriteTrigger(ColumnFamilyHandle* column_family) {
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
return -1;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
[RocksDB] BackupableDB
Summary:
In this diff I present you BackupableDB v1. You can easily use it to backup your DB and it will do incremental snapshots for you.
Let's first describe how you would use BackupableDB. It's inheriting StackableDB interface so you can easily construct it with your DB object -- it will add a method RollTheSnapshot() to the DB object. When you call RollTheSnapshot(), current snapshot of the DB will be stored in the backup dir. To restore, you can just call RestoreDBFromBackup() on a BackupableDB (which is a static method) and it will restore all files from the backup dir. In the next version, it will even support automatic backuping every X minutes.
There are multiple things you can configure:
1. backup_env and db_env can be different, which is awesome because then you can easily backup to HDFS or wherever you feel like.
2. sync - if true, it *guarantees* backup consistency on machine reboot
3. number of snapshots to keep - this will keep last N snapshots around if you want, for some reason, be able to restore from an earlier snapshot. All the backuping is done in incremental fashion - if we already have 00010.sst, we will not copy it again. *IMPORTANT* -- This is based on assumption that 00010.sst never changes - two files named 00010.sst from the same DB will always be exactly the same. Is this true? I always copy manifest, current and log files.
4. You can decide if you want to flush the memtables before you backup, or you're fine with backing up the log files -- either way, you get a complete and consistent view of the database at a time of backup.
5. More things you can find in BackupableDBOptions
Here is the directory structure I use:
backup_dir/CURRENT_SNAPSHOT - just 4 bytes holding the latest snapshot
0, 1, 2, ... - files containing serialized version of each snapshot - containing a list of files
files/*.sst - sst files shared between snapshots - if one snapshot references 00010.sst and another one needs to backup it from the DB, it will just reference the same file
files/ 0/, 1/, 2/, ... - snapshot directories containing private snapshot files - current, manifest and log files
All the files are ref counted and deleted immediatelly when they get out of scope.
Some other stuff in this diff:
1. Added GetEnv() method to the DB. Discussed with @haobo and we agreed that it seems right thing to do.
2. Fixed StackableDB interface. The way it was set up before, I was not able to implement BackupableDB.
Test Plan:
I have a unittest, but please don't look at this yet. I just hacked it up to help me with debugging. I will write a lot of good tests and update the diff.
Also, `make asan_check`
Reviewers: dhruba, haobo, emayanke
Reviewed By: dhruba
CC: leveldb, haobo
Differential Revision: https://reviews.facebook.net/D14295
2013-12-09 23:06:52 +01:00
|
|
|
virtual const std::string& GetName() const {
|
|
|
|
return name_;
|
|
|
|
}
|
|
|
|
|
2013-11-25 21:39:23 +01:00
|
|
|
virtual Env* GetEnv() const {
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::GetOptions;
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const {
|
2013-11-26 00:51:50 +01:00
|
|
|
return options_;
|
|
|
|
}
|
|
|
|
|
[RocksDB] [Column Family] Interface proposal
Summary:
<This diff is for Column Family branch>
Sharing some of the work I've done so far. This diff compiles and passes the tests.
The biggest change is in options.h - I broke down Options into two parts - DBOptions and ColumnFamilyOptions. DBOptions is DB-specific (env, create_if_missing, block_cache, etc.) and ColumnFamilyOptions is column family-specific (all compaction options, compresion options, etc.). Note that this does not break backwards compatibility at all.
Further, I created DBWithColumnFamily which inherits DB interface and adds new functions with column family support. Clients can transparently switch to DBWithColumnFamily and it will not break their backwards compatibility.
There are few methods worth checking out: ListColumnFamilies(), MultiNewIterator(), MultiGet() and GetSnapshot(). [GetSnapshot() returns the snapshot across all column families for now - I think that's what we agreed on]
Finally, I made small changes to WriteBatch so we are able to atomically insert data across column families.
Please provide feedback.
Test Plan: make check works, the code is backward compatible
Reviewers: dhruba, haobo, sdong, kailiu, emayanke
CC: leveldb
Differential Revision: https://reviews.facebook.net/D14445
2013-12-03 20:14:09 +01:00
|
|
|
using DB::Flush;
|
|
|
|
virtual Status Flush(const rocksdb::FlushOptions& options,
|
2014-02-11 02:04:44 +01:00
|
|
|
ColumnFamilyHandle* column_family) {
|
2012-07-06 20:42:09 +02:00
|
|
|
Status ret;
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
virtual Status DisableFileDeletions() {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2014-01-02 12:33:42 +01:00
|
|
|
virtual Status EnableFileDeletions(bool force) {
|
2012-09-15 02:11:35 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2013-10-03 23:38:32 +02:00
|
|
|
virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size,
|
|
|
|
bool flush_memtable = true) {
|
2012-09-15 02:11:35 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-08-06 21:54:37 +02:00
|
|
|
virtual Status GetSortedWalFiles(VectorLogPtr& files) {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-10-24 08:39:23 +02:00
|
|
|
virtual Status DeleteFile(std::string name) {
|
2013-08-06 21:54:37 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-12-05 19:16:39 +01:00
|
|
|
virtual Status GetDbIdentity(std::string& identity) {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2013-10-25 04:09:02 +02:00
|
|
|
virtual SequenceNumber GetLatestSequenceNumber() const {
|
2012-12-11 00:37:00 +01:00
|
|
|
return 0;
|
|
|
|
}
|
2014-02-28 20:50:36 +01:00
|
|
|
virtual Status GetUpdatesSince(
|
|
|
|
rocksdb::SequenceNumber, unique_ptr<rocksdb::TransactionLogIterator>*,
|
|
|
|
const TransactionLogIterator::ReadOptions&
|
|
|
|
read_options = TransactionLogIterator::ReadOptions()) {
|
2012-11-30 02:28:37 +01:00
|
|
|
return Status::NotSupported("Not supported in Model DB");
|
|
|
|
}
|
|
|
|
|
2014-02-11 02:04:44 +01:00
|
|
|
virtual ColumnFamilyHandle* DefaultColumnFamily() const { return nullptr; }
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
private:
|
|
|
|
class ModelIter: public Iterator {
|
|
|
|
public:
|
|
|
|
ModelIter(const KVMap* map, bool owned)
|
|
|
|
: map_(map), owned_(owned), iter_(map_->end()) {
|
|
|
|
}
|
|
|
|
~ModelIter() {
|
|
|
|
if (owned_) delete map_;
|
|
|
|
}
|
|
|
|
virtual bool Valid() const { return iter_ != map_->end(); }
|
|
|
|
virtual void SeekToFirst() { iter_ = map_->begin(); }
|
|
|
|
virtual void SeekToLast() {
|
|
|
|
if (map_->empty()) {
|
|
|
|
iter_ = map_->end();
|
|
|
|
} else {
|
|
|
|
iter_ = map_->find(map_->rbegin()->first);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
virtual void Seek(const Slice& k) {
|
|
|
|
iter_ = map_->lower_bound(k.ToString());
|
|
|
|
}
|
|
|
|
virtual void Next() { ++iter_; }
|
|
|
|
virtual void Prev() { --iter_; }
|
|
|
|
virtual Slice key() const { return iter_->first; }
|
|
|
|
virtual Slice value() const { return iter_->second; }
|
|
|
|
virtual Status status() const { return Status::OK(); }
|
|
|
|
private:
|
|
|
|
const KVMap* const map_;
|
|
|
|
const bool owned_; // Do we own map_
|
|
|
|
KVMap::const_iterator iter_;
|
|
|
|
};
|
|
|
|
const Options options_;
|
|
|
|
KVMap map_;
|
[RocksDB] BackupableDB
Summary:
In this diff I present you BackupableDB v1. You can easily use it to backup your DB and it will do incremental snapshots for you.
Let's first describe how you would use BackupableDB. It's inheriting StackableDB interface so you can easily construct it with your DB object -- it will add a method RollTheSnapshot() to the DB object. When you call RollTheSnapshot(), current snapshot of the DB will be stored in the backup dir. To restore, you can just call RestoreDBFromBackup() on a BackupableDB (which is a static method) and it will restore all files from the backup dir. In the next version, it will even support automatic backuping every X minutes.
There are multiple things you can configure:
1. backup_env and db_env can be different, which is awesome because then you can easily backup to HDFS or wherever you feel like.
2. sync - if true, it *guarantees* backup consistency on machine reboot
3. number of snapshots to keep - this will keep last N snapshots around if you want, for some reason, be able to restore from an earlier snapshot. All the backuping is done in incremental fashion - if we already have 00010.sst, we will not copy it again. *IMPORTANT* -- This is based on assumption that 00010.sst never changes - two files named 00010.sst from the same DB will always be exactly the same. Is this true? I always copy manifest, current and log files.
4. You can decide if you want to flush the memtables before you backup, or you're fine with backing up the log files -- either way, you get a complete and consistent view of the database at a time of backup.
5. More things you can find in BackupableDBOptions
Here is the directory structure I use:
backup_dir/CURRENT_SNAPSHOT - just 4 bytes holding the latest snapshot
0, 1, 2, ... - files containing serialized version of each snapshot - containing a list of files
files/*.sst - sst files shared between snapshots - if one snapshot references 00010.sst and another one needs to backup it from the DB, it will just reference the same file
files/ 0/, 1/, 2/, ... - snapshot directories containing private snapshot files - current, manifest and log files
All the files are ref counted and deleted immediatelly when they get out of scope.
Some other stuff in this diff:
1. Added GetEnv() method to the DB. Discussed with @haobo and we agreed that it seems right thing to do.
2. Fixed StackableDB interface. The way it was set up before, I was not able to implement BackupableDB.
Test Plan:
I have a unittest, but please don't look at this yet. I just hacked it up to help me with debugging. I will write a lot of good tests and update the diff.
Also, `make asan_check`
Reviewers: dhruba, haobo, emayanke
Reviewed By: dhruba
CC: leveldb, haobo
Differential Revision: https://reviews.facebook.net/D14295
2013-12-09 23:06:52 +01:00
|
|
|
std::string name_ = "";
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
2013-08-23 08:10:02 +02:00
|
|
|
static std::string RandomKey(Random* rnd, int minimum = 0) {
|
|
|
|
int len;
|
|
|
|
do {
|
|
|
|
len = (rnd->OneIn(3)
|
|
|
|
? 1 // Short sometimes to encourage collisions
|
|
|
|
: (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
|
|
|
|
} while (len < minimum);
|
2011-03-18 23:37:00 +01:00
|
|
|
return test::RandomKey(rnd, len);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool CompareIterators(int step,
|
|
|
|
DB* model,
|
|
|
|
DB* db,
|
|
|
|
const Snapshot* model_snap,
|
|
|
|
const Snapshot* db_snap) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.snapshot = model_snap;
|
|
|
|
Iterator* miter = model->NewIterator(options);
|
|
|
|
options.snapshot = db_snap;
|
|
|
|
Iterator* dbiter = db->NewIterator(options);
|
|
|
|
bool ok = true;
|
|
|
|
int count = 0;
|
|
|
|
for (miter->SeekToFirst(), dbiter->SeekToFirst();
|
|
|
|
ok && miter->Valid() && dbiter->Valid();
|
|
|
|
miter->Next(), dbiter->Next()) {
|
|
|
|
count++;
|
|
|
|
if (miter->key().compare(dbiter->key()) != 0) {
|
|
|
|
fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
|
|
|
|
step,
|
|
|
|
EscapeString(miter->key()).c_str(),
|
|
|
|
EscapeString(dbiter->key()).c_str());
|
|
|
|
ok = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (miter->value().compare(dbiter->value()) != 0) {
|
|
|
|
fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
|
|
|
|
step,
|
|
|
|
EscapeString(miter->key()).c_str(),
|
|
|
|
EscapeString(miter->value()).c_str(),
|
|
|
|
EscapeString(miter->value()).c_str());
|
|
|
|
ok = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ok) {
|
|
|
|
if (miter->Valid() != dbiter->Valid()) {
|
|
|
|
fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
|
|
|
|
step, miter->Valid(), dbiter->Valid());
|
|
|
|
ok = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
delete miter;
|
|
|
|
delete dbiter;
|
|
|
|
return ok;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, Randomized) {
|
|
|
|
Random rnd(test::RandomSeed());
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ModelDB model(CurrentOptions());
|
|
|
|
const int N = 10000;
|
2013-03-01 03:04:58 +01:00
|
|
|
const Snapshot* model_snap = nullptr;
|
|
|
|
const Snapshot* db_snap = nullptr;
|
2012-04-17 17:36:46 +02:00
|
|
|
std::string k, v;
|
|
|
|
for (int step = 0; step < N; step++) {
|
|
|
|
// TODO(sanjay): Test Get() works
|
|
|
|
int p = rnd.Uniform(100);
|
2013-08-23 08:10:02 +02:00
|
|
|
int minimum = 0;
|
2013-12-20 18:35:24 +01:00
|
|
|
if (option_config_ == kHashSkipList ||
|
2013-12-27 21:56:27 +01:00
|
|
|
option_config_ == kHashLinkList ||
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
option_config_ == kHashCuckoo ||
|
2014-04-10 23:19:43 +02:00
|
|
|
option_config_ == kPlainTableFirstBytePrefix ||
|
|
|
|
option_config_ == kBlockBasedTableWithWholeKeyHashIndex ||
|
|
|
|
option_config_ == kBlockBasedTableWithPrefixHashIndex) {
|
2013-08-23 08:10:02 +02:00
|
|
|
minimum = 1;
|
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
if (p < 45) { // Put
|
2013-08-23 08:10:02 +02:00
|
|
|
k = RandomKey(&rnd, minimum);
|
2012-04-17 17:36:46 +02:00
|
|
|
v = RandomString(&rnd,
|
|
|
|
rnd.OneIn(20)
|
|
|
|
? 100 + rnd.Uniform(100)
|
|
|
|
: rnd.Uniform(8));
|
|
|
|
ASSERT_OK(model.Put(WriteOptions(), k, v));
|
|
|
|
ASSERT_OK(db_->Put(WriteOptions(), k, v));
|
|
|
|
|
|
|
|
} else if (p < 90) { // Delete
|
2013-08-23 08:10:02 +02:00
|
|
|
k = RandomKey(&rnd, minimum);
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_OK(model.Delete(WriteOptions(), k));
|
|
|
|
ASSERT_OK(db_->Delete(WriteOptions(), k));
|
|
|
|
|
|
|
|
|
|
|
|
} else { // Multi-element batch
|
|
|
|
WriteBatch b;
|
|
|
|
const int num = rnd.Uniform(8);
|
|
|
|
for (int i = 0; i < num; i++) {
|
|
|
|
if (i == 0 || !rnd.OneIn(10)) {
|
2013-08-23 08:10:02 +02:00
|
|
|
k = RandomKey(&rnd, minimum);
|
2012-04-17 17:36:46 +02:00
|
|
|
} else {
|
|
|
|
// Periodically re-use the same key from the previous iter, so
|
|
|
|
// we have multiple entries in the write batch for the same key
|
|
|
|
}
|
|
|
|
if (rnd.OneIn(2)) {
|
|
|
|
v = RandomString(&rnd, rnd.Uniform(10));
|
|
|
|
b.Put(k, v);
|
|
|
|
} else {
|
|
|
|
b.Delete(k);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_OK(model.Write(WriteOptions(), &b));
|
|
|
|
ASSERT_OK(db_->Write(WriteOptions(), &b));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
if ((step % 100) == 0) {
|
2014-04-11 02:16:23 +02:00
|
|
|
ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr));
|
|
|
|
ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
|
2014-04-10 23:19:43 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Save a snapshot from each DB this time that we'll use next
|
|
|
|
// time we compare things, to make sure the current state is
|
|
|
|
// preserved with the snapshot
|
2013-03-01 03:04:58 +01:00
|
|
|
if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
|
|
|
|
if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen();
|
2013-03-01 03:04:58 +01:00
|
|
|
ASSERT_TRUE(CompareIterators(step, &model, db_, nullptr, nullptr));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
model_snap = model.GetSnapshot();
|
|
|
|
db_snap = db_->GetSnapshot();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2013-03-01 03:04:58 +01:00
|
|
|
if (model_snap != nullptr) model.ReleaseSnapshot(model_snap);
|
|
|
|
if (db_snap != nullptr) db_->ReleaseSnapshot(db_snap);
|
Add a new mem-table representation based on cuckoo hash.
Summary:
= Major Changes =
* Add a new mem-table representation, HashCuckooRep, which is based cuckoo hash.
Cuckoo hash uses multiple hash functions. This allows each key to have multiple
possible locations in the mem-table.
- Put: When insert a key, it will try to find whether one of its possible
locations is vacant and store the key. If none of its possible
locations are available, then it will kick out a victim key and
store at that location. The kicked-out victim key will then be
stored at a vacant space of its possible locations or kick-out
another victim. In this diff, the kick-out path (known as
cuckoo-path) is found using BFS, which guarantees to be the shortest.
- Get: Simply tries all possible locations of a key --- this guarantees
worst-case constant time complexity.
- Time complexity: O(1) for Get, and average O(1) for Put if the
fullness of the mem-table is below 80%.
- Default using two hash functions, the number of hash functions used
by the cuckoo-hash may dynamically increase if it fails to find a
short-enough kick-out path.
- Currently, HashCuckooRep does not support iteration and snapshots,
as our current main purpose of this is to optimize point access.
= Minor Changes =
* Add IsSnapshotSupported() to DB to indicate whether the current DB
supports snapshots. If it returns false, then DB::GetSnapshot() will
always return nullptr.
Test Plan:
Run existing tests. Will develop a test specifically for cuckoo hash in
the next diff.
Reviewers: sdong, haobo
Reviewed By: sdong
CC: leveldb, dhruba, igor
Differential Revision: https://reviews.facebook.net/D16155
2014-04-30 02:13:46 +02:00
|
|
|
// skip cuckoo hash as it does not support snapshot.
|
|
|
|
} while (ChangeOptions(kSkipDeletesFilterFirst |
|
|
|
|
kSkipNoSeekToLast | kSkipHashCuckoo));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2013-06-05 20:22:38 +02:00
|
|
|
TEST(DBTest, MultiGetSimple) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ASSERT_OK(Put(1, "k1", "v1"));
|
|
|
|
ASSERT_OK(Put(1, "k2", "v2"));
|
|
|
|
ASSERT_OK(Put(1, "k3", "v3"));
|
|
|
|
ASSERT_OK(Put(1, "k4", "v4"));
|
|
|
|
ASSERT_OK(Delete(1, "k4"));
|
|
|
|
ASSERT_OK(Put(1, "k5", "v5"));
|
|
|
|
ASSERT_OK(Delete(1, "no_key"));
|
|
|
|
|
|
|
|
std::vector<Slice> keys({"k1", "k2", "k3", "k4", "k5", "no_key"});
|
|
|
|
|
|
|
|
std::vector<std::string> values(20, "Temporary data to be overwritten");
|
|
|
|
std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
|
|
|
|
|
|
|
|
std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
|
|
|
|
ASSERT_EQ(values.size(), keys.size());
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_EQ(values[0], "v1");
|
|
|
|
ASSERT_EQ(values[1], "v2");
|
|
|
|
ASSERT_EQ(values[2], "v3");
|
|
|
|
ASSERT_EQ(values[4], "v5");
|
|
|
|
|
|
|
|
ASSERT_OK(s[0]);
|
|
|
|
ASSERT_OK(s[1]);
|
|
|
|
ASSERT_OK(s[2]);
|
|
|
|
ASSERT_TRUE(s[3].IsNotFound());
|
|
|
|
ASSERT_OK(s[4]);
|
|
|
|
ASSERT_TRUE(s[5].IsNotFound());
|
|
|
|
} while (ChangeCompactOptions());
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, MultiGetEmpty) {
|
2013-08-08 00:20:41 +02:00
|
|
|
do {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2013-08-08 00:20:41 +02:00
|
|
|
// Empty Key Set
|
|
|
|
std::vector<Slice> keys;
|
|
|
|
std::vector<std::string> values;
|
2014-02-07 23:47:16 +01:00
|
|
|
std::vector<ColumnFamilyHandle*> cfs;
|
|
|
|
std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
|
|
|
|
ASSERT_EQ(s.size(), 0U);
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
// Empty Database, Empty Key Set
|
|
|
|
DestroyAndReopen();
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
|
|
|
|
ASSERT_EQ(s.size(), 0U);
|
2013-08-08 00:20:41 +02:00
|
|
|
|
|
|
|
// Empty Database, Search for Keys
|
|
|
|
keys.resize(2);
|
|
|
|
keys[0] = "a";
|
|
|
|
keys[1] = "b";
|
2014-02-07 23:47:16 +01:00
|
|
|
cfs.push_back(handles_[0]);
|
|
|
|
cfs.push_back(handles_[1]);
|
|
|
|
s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
|
2013-08-08 00:20:41 +02:00
|
|
|
ASSERT_EQ((int)s.size(), 2);
|
|
|
|
ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound());
|
|
|
|
} while (ChangeCompactOptions());
|
2013-06-05 20:22:38 +02:00
|
|
|
}
|
|
|
|
|
2014-04-10 06:17:14 +02:00
|
|
|
namespace {
|
2013-08-13 23:04:56 +02:00
|
|
|
void PrefixScanInit(DBTest *dbtest) {
|
|
|
|
char buf[100];
|
|
|
|
std::string keystr;
|
|
|
|
const int small_range_sstfiles = 5;
|
|
|
|
const int big_range_sstfiles = 5;
|
|
|
|
|
|
|
|
// Generate 11 sst files with the following prefix ranges.
|
|
|
|
// GROUP 0: [0,10] (level 1)
|
|
|
|
// GROUP 1: [1,2], [2,3], [3,4], [4,5], [5, 6] (level 0)
|
|
|
|
// GROUP 2: [0,6], [0,7], [0,8], [0,9], [0,10] (level 0)
|
|
|
|
//
|
|
|
|
// A seek with the previous API would do 11 random I/Os (to all the
|
|
|
|
// files). With the new API and a prefix filter enabled, we should
|
|
|
|
// only do 2 random I/O, to the 2 files containing the key.
|
|
|
|
|
|
|
|
// GROUP 0
|
|
|
|
snprintf(buf, sizeof(buf), "%02d______:start", 0);
|
|
|
|
keystr = std::string(buf);
|
|
|
|
ASSERT_OK(dbtest->Put(keystr, keystr));
|
|
|
|
snprintf(buf, sizeof(buf), "%02d______:end", 10);
|
|
|
|
keystr = std::string(buf);
|
|
|
|
ASSERT_OK(dbtest->Put(keystr, keystr));
|
2014-02-07 23:47:16 +01:00
|
|
|
dbtest->Flush();
|
2013-08-13 23:04:56 +02:00
|
|
|
dbtest->dbfull()->CompactRange(nullptr, nullptr); // move to level 1
|
|
|
|
|
|
|
|
// GROUP 1
|
|
|
|
for (int i = 1; i <= small_range_sstfiles; i++) {
|
|
|
|
snprintf(buf, sizeof(buf), "%02d______:start", i);
|
|
|
|
keystr = std::string(buf);
|
|
|
|
ASSERT_OK(dbtest->Put(keystr, keystr));
|
|
|
|
snprintf(buf, sizeof(buf), "%02d______:end", i+1);
|
|
|
|
keystr = std::string(buf);
|
|
|
|
ASSERT_OK(dbtest->Put(keystr, keystr));
|
2014-02-07 23:47:16 +01:00
|
|
|
dbtest->Flush();
|
2013-08-13 23:04:56 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// GROUP 2
|
|
|
|
for (int i = 1; i <= big_range_sstfiles; i++) {
|
|
|
|
std::string keystr;
|
|
|
|
snprintf(buf, sizeof(buf), "%02d______:start", 0);
|
|
|
|
keystr = std::string(buf);
|
|
|
|
ASSERT_OK(dbtest->Put(keystr, keystr));
|
|
|
|
snprintf(buf, sizeof(buf), "%02d______:end",
|
|
|
|
small_range_sstfiles+i+1);
|
|
|
|
keystr = std::string(buf);
|
|
|
|
ASSERT_OK(dbtest->Put(keystr, keystr));
|
2014-02-07 23:47:16 +01:00
|
|
|
dbtest->Flush();
|
2013-08-13 23:04:56 +02:00
|
|
|
}
|
|
|
|
}
|
2014-04-10 06:17:14 +02:00
|
|
|
} // namespace
|
2013-08-13 23:04:56 +02:00
|
|
|
|
|
|
|
TEST(DBTest, PrefixScan) {
|
2013-12-03 21:42:15 +01:00
|
|
|
int count;
|
|
|
|
Slice prefix;
|
|
|
|
Slice key;
|
|
|
|
char buf[100];
|
|
|
|
Iterator* iter;
|
|
|
|
snprintf(buf, sizeof(buf), "03______:");
|
|
|
|
prefix = Slice(buf, 8);
|
|
|
|
key = Slice(buf, 9);
|
|
|
|
// db configs
|
|
|
|
env_->count_random_reads_ = true;
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.no_block_cache = true;
|
2014-01-03 19:53:21 +01:00
|
|
|
options.filter_policy = NewBloomFilterPolicy(10);
|
2014-03-10 20:56:46 +01:00
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
2013-12-03 21:42:15 +01:00
|
|
|
options.whole_key_filtering = false;
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
options.max_background_compactions = 2;
|
|
|
|
options.create_if_missing = true;
|
2014-07-01 00:54:31 +02:00
|
|
|
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
|
2013-08-13 23:04:56 +02:00
|
|
|
|
2014-04-25 21:21:34 +02:00
|
|
|
// 11 RAND I/Os
|
2013-12-03 21:42:15 +01:00
|
|
|
DestroyAndReopen(&options);
|
|
|
|
PrefixScanInit(this);
|
|
|
|
count = 0;
|
|
|
|
env_->random_read_counter_.Reset();
|
|
|
|
iter = db_->NewIterator(ReadOptions());
|
|
|
|
for (iter->Seek(prefix); iter->Valid(); iter->Next()) {
|
|
|
|
if (! iter->key().starts_with(prefix)) {
|
|
|
|
break;
|
2013-08-13 23:04:56 +02:00
|
|
|
}
|
2013-12-03 21:42:15 +01:00
|
|
|
count++;
|
2013-08-13 23:04:56 +02:00
|
|
|
}
|
2013-12-03 21:42:15 +01:00
|
|
|
ASSERT_OK(iter->status());
|
|
|
|
delete iter;
|
|
|
|
ASSERT_EQ(count, 2);
|
2014-04-25 21:22:23 +02:00
|
|
|
ASSERT_EQ(env_->random_read_counter_.Read(), 2);
|
2013-12-03 21:42:15 +01:00
|
|
|
Close();
|
|
|
|
delete options.filter_policy;
|
2013-08-13 23:04:56 +02:00
|
|
|
}
|
|
|
|
|
2014-01-17 06:56:26 +01:00
|
|
|
TEST(DBTest, TailingIteratorSingle) {
|
|
|
|
ReadOptions read_options;
|
|
|
|
read_options.tailing = true;
|
|
|
|
|
|
|
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
|
|
|
|
// add a record and check that iter can see it
|
|
|
|
ASSERT_OK(db_->Put(WriteOptions(), "mirko", "fodor"));
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ(iter->key().ToString(), "mirko");
|
|
|
|
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, TailingIteratorKeepAdding) {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2014-01-17 06:56:26 +01:00
|
|
|
ReadOptions read_options;
|
|
|
|
read_options.tailing = true;
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, handles_[1]));
|
2014-01-17 06:56:26 +01:00
|
|
|
std::string value(1024, 'a');
|
|
|
|
|
|
|
|
const int num_records = 10000;
|
|
|
|
for (int i = 0; i < num_records; ++i) {
|
|
|
|
char buf[32];
|
|
|
|
snprintf(buf, sizeof(buf), "%016d", i);
|
|
|
|
|
|
|
|
Slice key(buf, 16);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, key, value));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
|
|
|
iter->Seek(key);
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ(iter->key().compare(key), 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-06-10 18:57:26 +02:00
|
|
|
TEST(DBTest, TailingIteratorSeekToNext) {
|
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
|
|
|
ReadOptions read_options;
|
|
|
|
read_options.tailing = true;
|
|
|
|
|
|
|
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, handles_[1]));
|
|
|
|
std::string value(1024, 'a');
|
|
|
|
|
|
|
|
const int num_records = 1000;
|
|
|
|
for (int i = 1; i < num_records; ++i) {
|
|
|
|
char buf1[32];
|
|
|
|
char buf2[32];
|
|
|
|
snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5);
|
|
|
|
|
|
|
|
Slice key(buf1, 20);
|
|
|
|
ASSERT_OK(Put(1, key, value));
|
|
|
|
|
|
|
|
if (i % 100 == 99) {
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
}
|
|
|
|
|
|
|
|
snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2);
|
|
|
|
Slice target(buf2, 20);
|
|
|
|
iter->Seek(target);
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ(iter->key().compare(key), 0);
|
|
|
|
}
|
|
|
|
for (int i = 2 * num_records; i > 0; --i) {
|
|
|
|
char buf1[32];
|
|
|
|
char buf2[32];
|
|
|
|
snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5);
|
|
|
|
|
|
|
|
Slice key(buf1, 20);
|
|
|
|
ASSERT_OK(Put(1, key, value));
|
|
|
|
|
|
|
|
if (i % 100 == 99) {
|
|
|
|
ASSERT_OK(Flush(1));
|
|
|
|
}
|
|
|
|
|
|
|
|
snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2);
|
|
|
|
Slice target(buf2, 20);
|
|
|
|
iter->Seek(target);
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ(iter->key().compare(key), 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-01-17 06:56:26 +01:00
|
|
|
TEST(DBTest, TailingIteratorDeletes) {
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"});
|
2014-01-17 06:56:26 +01:00
|
|
|
ReadOptions read_options;
|
|
|
|
read_options.tailing = true;
|
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, handles_[1]));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
|
|
|
// write a single record, read it using the iterator, then delete it
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "0test", "test"));
|
2014-01-17 06:56:26 +01:00
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ(iter->key().ToString(), "0test");
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Delete(1, "0test"));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
|
|
|
// write many more records
|
|
|
|
const int num_records = 10000;
|
|
|
|
std::string value(1024, 'A');
|
|
|
|
|
|
|
|
for (int i = 0; i < num_records; ++i) {
|
|
|
|
char buf[32];
|
|
|
|
snprintf(buf, sizeof(buf), "1%015d", i);
|
|
|
|
|
|
|
|
Slice key(buf, 16);
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, key, value));
|
2014-01-17 06:56:26 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// force a flush to make sure that no records are read from memtable
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
|
|
|
// skip "0test"
|
|
|
|
iter->Next();
|
|
|
|
|
|
|
|
// make sure we can read all new records using the existing iterator
|
|
|
|
int count = 0;
|
|
|
|
for (; iter->Valid(); iter->Next(), ++count) ;
|
|
|
|
|
|
|
|
ASSERT_EQ(count, num_records);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, TailingIteratorPrefixSeek) {
|
|
|
|
ReadOptions read_options;
|
|
|
|
read_options.tailing = true;
|
|
|
|
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.disable_auto_compactions = true;
|
2014-03-10 20:56:46 +01:00
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(2));
|
2014-07-01 00:54:31 +02:00
|
|
|
options.memtable_factory.reset(NewHashSkipListRepFactory(16));
|
2014-01-17 06:56:26 +01:00
|
|
|
DestroyAndReopen(&options);
|
2014-02-07 23:47:16 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, &options);
|
2014-01-17 06:56:26 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, handles_[1]));
|
|
|
|
ASSERT_OK(Put(1, "0101", "test"));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Flush(1));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
2014-02-07 23:47:16 +01:00
|
|
|
ASSERT_OK(Put(1, "0202", "test"));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
|
|
|
// Seek(0102) shouldn't find any records since 0202 has a different prefix
|
|
|
|
iter->Seek("0102");
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
|
|
|
|
iter->Seek("0202");
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ(iter->key().ToString(), "0202");
|
|
|
|
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
}
|
|
|
|
|
2014-06-20 00:32:31 +02:00
|
|
|
TEST(DBTest, BlockBasedTablePrefixIndexTest) {
|
|
|
|
// create a DB with block prefix index
|
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kHashSearch;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
|
|
|
|
|
|
|
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_OK(Put("k1", "v1"));
|
|
|
|
Flush();
|
|
|
|
ASSERT_OK(Put("k2", "v2"));
|
|
|
|
|
|
|
|
// Reopen it without prefix extractor, make sure everything still works.
|
|
|
|
// RocksDB should just fall back to the binary index.
|
|
|
|
table_options.index_type = BlockBasedTableOptions::kBinarySearch;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
options.prefix_extractor.reset();
|
|
|
|
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ("v1", Get("k1"));
|
|
|
|
ASSERT_EQ("v2", Get("k2"));
|
|
|
|
}
|
|
|
|
|
2014-05-01 20:09:32 +02:00
|
|
|
TEST(DBTest, ChecksumTest) {
|
|
|
|
BlockBasedTableOptions table_options;
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
|
|
|
|
table_options.checksum = kCRC32c;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_OK(Put("a", "b"));
|
|
|
|
ASSERT_OK(Put("c", "d"));
|
|
|
|
ASSERT_OK(Flush()); // table with crc checksum
|
|
|
|
|
|
|
|
table_options.checksum = kxxHash;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_OK(Put("e", "f"));
|
|
|
|
ASSERT_OK(Put("g", "h"));
|
|
|
|
ASSERT_OK(Flush()); // table with xxhash checksum
|
|
|
|
|
|
|
|
table_options.checksum = kCRC32c;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ("b", Get("a"));
|
|
|
|
ASSERT_EQ("d", Get("c"));
|
|
|
|
ASSERT_EQ("f", Get("e"));
|
|
|
|
ASSERT_EQ("h", Get("g"));
|
2014-01-17 06:56:26 +01:00
|
|
|
|
2014-05-01 20:09:32 +02:00
|
|
|
table_options.checksum = kCRC32c;
|
|
|
|
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ("b", Get("a"));
|
|
|
|
ASSERT_EQ("d", Get("c"));
|
|
|
|
ASSERT_EQ("f", Get("e"));
|
|
|
|
ASSERT_EQ("h", Get("g"));
|
|
|
|
}
|
2014-05-21 20:43:35 +02:00
|
|
|
|
|
|
|
TEST(DBTest, FIFOCompactionTest) {
|
|
|
|
for (int iter = 0; iter < 2; ++iter) {
|
|
|
|
// first iteration -- auto compaction
|
|
|
|
// second iteration -- manual compaction
|
|
|
|
Options options;
|
|
|
|
options.compaction_style = kCompactionStyleFIFO;
|
|
|
|
options.write_buffer_size = 100 << 10; // 100KB
|
|
|
|
options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
if (iter == 1) {
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
}
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
for (int i = 0; i < 6; ++i) {
|
|
|
|
for (int j = 0; j < 100; ++j) {
|
|
|
|
ASSERT_OK(Put(std::to_string(i * 100 + j), RandomString(&rnd, 1024)));
|
|
|
|
}
|
|
|
|
// flush should happen here
|
|
|
|
}
|
|
|
|
if (iter == 0) {
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
|
|
} else {
|
|
|
|
ASSERT_OK(db_->CompactRange(nullptr, nullptr));
|
|
|
|
}
|
|
|
|
// only 5 files should survive
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
|
|
|
|
for (int i = 0; i < 50; ++i) {
|
|
|
|
// these keys should be deleted in previous compaction
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(std::to_string(i)));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
2013-10-04 06:49:15 +02:00
|
|
|
return rocksdb::test::RunAllTests();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|