2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
2012-09-24 23:01:01 +02:00
|
|
|
#include <algorithm>
|
2012-11-26 22:56:45 +01:00
|
|
|
#include <set>
|
|
|
|
|
2011-03-30 20:35:40 +02:00
|
|
|
#include "leveldb/db.h"
|
2012-04-17 17:36:46 +02:00
|
|
|
#include "leveldb/filter_policy.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/db_impl.h"
|
|
|
|
#include "db/filename.h"
|
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "db/write_batch_internal.h"
|
2012-04-17 17:36:46 +02:00
|
|
|
#include "leveldb/cache.h"
|
2011-03-30 20:35:40 +02:00
|
|
|
#include "leveldb/env.h"
|
|
|
|
#include "leveldb/table.h"
|
2012-04-17 17:36:46 +02:00
|
|
|
#include "util/hash.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/logging.h"
|
2011-09-01 21:08:02 +02:00
|
|
|
#include "util/mutexlock.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "util/testharness.h"
|
|
|
|
#include "util/testutil.h"
|
|
|
|
|
|
|
|
namespace leveldb {
|
|
|
|
|
2012-11-01 18:50:08 +01:00
|
|
|
static bool SnappyCompressionSupported(const CompressionOptions& options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::Snappy_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool ZlibCompressionSupported(const CompressionOptions& options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::Zlib_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool BZip2CompressionSupported(const CompressionOptions& options) {
|
|
|
|
std::string out;
|
|
|
|
Slice in = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
return port::BZip2_Compress(options, in.data(), in.size(), &out);
|
|
|
|
}
|
2012-10-28 07:13:17 +01:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
static std::string RandomString(Random* rnd, int len) {
|
|
|
|
std::string r;
|
|
|
|
test::RandomString(rnd, len, &r);
|
|
|
|
return r;
|
|
|
|
}
|
|
|
|
|
2012-11-06 21:02:18 +01:00
|
|
|
namespace anon {
|
2012-04-17 17:36:46 +02:00
|
|
|
class AtomicCounter {
|
|
|
|
private:
|
|
|
|
port::Mutex mu_;
|
|
|
|
int count_;
|
|
|
|
public:
|
|
|
|
AtomicCounter() : count_(0) { }
|
|
|
|
void Increment() {
|
|
|
|
MutexLock l(&mu_);
|
|
|
|
count_++;
|
|
|
|
}
|
|
|
|
int Read() {
|
|
|
|
MutexLock l(&mu_);
|
|
|
|
return count_;
|
|
|
|
}
|
|
|
|
void Reset() {
|
|
|
|
MutexLock l(&mu_);
|
|
|
|
count_ = 0;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Special Env used to delay background operations
|
|
|
|
class SpecialEnv : public EnvWrapper {
|
|
|
|
public:
|
|
|
|
// sstable Sync() calls are blocked while this pointer is non-NULL.
|
|
|
|
port::AtomicPointer delay_sstable_sync_;
|
|
|
|
|
2012-01-25 23:56:52 +01:00
|
|
|
// Simulate no-space errors while this pointer is non-NULL.
|
|
|
|
port::AtomicPointer no_space_;
|
|
|
|
|
2012-08-23 01:57:51 +02:00
|
|
|
// Simulate non-writable file system while this pointer is non-NULL
|
|
|
|
port::AtomicPointer non_writable_;
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
bool count_random_reads_;
|
2012-11-06 21:02:18 +01:00
|
|
|
anon::AtomicCounter random_read_counter_;
|
2012-04-17 17:36:46 +02:00
|
|
|
|
2012-11-06 21:02:18 +01:00
|
|
|
anon::AtomicCounter sleep_counter_;
|
2012-08-23 01:57:51 +02:00
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
explicit SpecialEnv(Env* base) : EnvWrapper(base) {
|
|
|
|
delay_sstable_sync_.Release_Store(NULL);
|
2012-01-25 23:56:52 +01:00
|
|
|
no_space_.Release_Store(NULL);
|
2012-08-23 01:57:51 +02:00
|
|
|
non_writable_.Release_Store(NULL);
|
2012-04-17 17:36:46 +02:00
|
|
|
count_random_reads_ = false;
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
Status NewWritableFile(const std::string& f, WritableFile** r) {
|
|
|
|
class SSTableFile : public WritableFile {
|
|
|
|
private:
|
|
|
|
SpecialEnv* env_;
|
|
|
|
WritableFile* base_;
|
|
|
|
|
|
|
|
public:
|
|
|
|
SSTableFile(SpecialEnv* env, WritableFile* base)
|
|
|
|
: env_(env),
|
|
|
|
base_(base) {
|
|
|
|
}
|
2011-07-20 01:36:47 +02:00
|
|
|
~SSTableFile() { delete base_; }
|
2012-01-25 23:56:52 +01:00
|
|
|
Status Append(const Slice& data) {
|
|
|
|
if (env_->no_space_.Acquire_Load() != NULL) {
|
|
|
|
// Drop writes on the floor
|
|
|
|
return Status::OK();
|
|
|
|
} else {
|
|
|
|
return base_->Append(data);
|
|
|
|
}
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
Status Close() { return base_->Close(); }
|
|
|
|
Status Flush() { return base_->Flush(); }
|
|
|
|
Status Sync() {
|
|
|
|
while (env_->delay_sstable_sync_.Acquire_Load() != NULL) {
|
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
|
|
|
return base_->Sync();
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2012-08-23 01:57:51 +02:00
|
|
|
if (non_writable_.Acquire_Load() != NULL) {
|
|
|
|
return Status::IOError("simulated write error");
|
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
Status s = target()->NewWritableFile(f, r);
|
|
|
|
if (s.ok()) {
|
|
|
|
if (strstr(f.c_str(), ".sst") != NULL) {
|
|
|
|
*r = new SSTableFile(this, *r);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
|
|
|
|
Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
|
|
|
|
class CountingFile : public RandomAccessFile {
|
|
|
|
private:
|
|
|
|
RandomAccessFile* target_;
|
2012-11-06 21:02:18 +01:00
|
|
|
anon::AtomicCounter* counter_;
|
2012-04-17 17:36:46 +02:00
|
|
|
public:
|
2012-11-06 21:02:18 +01:00
|
|
|
CountingFile(RandomAccessFile* target, anon::AtomicCounter* counter)
|
2012-04-17 17:36:46 +02:00
|
|
|
: target_(target), counter_(counter) {
|
|
|
|
}
|
|
|
|
virtual ~CountingFile() { delete target_; }
|
|
|
|
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
|
|
|
char* scratch) const {
|
|
|
|
counter_->Increment();
|
|
|
|
return target_->Read(offset, n, result, scratch);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
Status s = target()->NewRandomAccessFile(f, r);
|
|
|
|
if (s.ok() && count_random_reads_) {
|
|
|
|
*r = new CountingFile(*r, &random_read_counter_);
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
2012-08-23 01:57:51 +02:00
|
|
|
|
|
|
|
virtual void SleepForMicroseconds(int micros) {
|
|
|
|
sleep_counter_.Increment();
|
|
|
|
target()->SleepForMicroseconds(micros);
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
};
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
class DBTest {
|
2012-04-17 17:36:46 +02:00
|
|
|
private:
|
|
|
|
const FilterPolicy* filter_policy_;
|
|
|
|
|
|
|
|
// Sequence of option configurations to try
|
|
|
|
enum OptionConfig {
|
|
|
|
kDefault,
|
|
|
|
kFilter,
|
2012-08-27 08:45:35 +02:00
|
|
|
kUncompressed,
|
2012-06-23 04:30:03 +02:00
|
|
|
kNumLevel_3,
|
2012-09-06 02:44:13 +02:00
|
|
|
kDBLogDir,
|
2012-04-17 17:36:46 +02:00
|
|
|
kEnd
|
|
|
|
};
|
|
|
|
int option_config_;
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
public:
|
|
|
|
std::string dbname_;
|
2011-06-22 04:36:45 +02:00
|
|
|
SpecialEnv* env_;
|
2011-03-18 23:37:00 +01:00
|
|
|
DB* db_;
|
|
|
|
|
|
|
|
Options last_options_;
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
DBTest() : option_config_(kDefault),
|
|
|
|
env_(new SpecialEnv(Env::Default())) {
|
|
|
|
filter_policy_ = NewBloomFilterPolicy(10);
|
2011-03-18 23:37:00 +01:00
|
|
|
dbname_ = test::TmpDir() + "/db_test";
|
|
|
|
DestroyDB(dbname_, Options());
|
|
|
|
db_ = NULL;
|
|
|
|
Reopen();
|
|
|
|
}
|
|
|
|
|
|
|
|
~DBTest() {
|
|
|
|
delete db_;
|
|
|
|
DestroyDB(dbname_, Options());
|
2011-06-22 04:36:45 +02:00
|
|
|
delete env_;
|
2012-04-17 17:36:46 +02:00
|
|
|
delete filter_policy_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Switch to a fresh database with the next option configuration to
|
|
|
|
// test. Return false if there are no more configurations to test.
|
|
|
|
bool ChangeOptions() {
|
2012-08-27 08:45:35 +02:00
|
|
|
option_config_++;
|
|
|
|
if (option_config_ >= kEnd) {
|
2012-04-17 17:36:46 +02:00
|
|
|
return false;
|
|
|
|
} else {
|
|
|
|
DestroyAndReopen();
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return the current option configuration.
|
|
|
|
Options CurrentOptions() {
|
|
|
|
Options options;
|
|
|
|
switch (option_config_) {
|
|
|
|
case kFilter:
|
|
|
|
options.filter_policy = filter_policy_;
|
|
|
|
break;
|
2012-08-27 08:45:35 +02:00
|
|
|
case kUncompressed:
|
2012-09-06 02:44:13 +02:00
|
|
|
options.compression = kNoCompression;
|
|
|
|
break;
|
2012-06-23 04:30:03 +02:00
|
|
|
case kNumLevel_3:
|
2012-09-06 02:44:13 +02:00
|
|
|
options.num_levels = 3;
|
|
|
|
break;
|
|
|
|
case kDBLogDir:
|
|
|
|
options.db_log_dir = test::TmpDir();
|
|
|
|
break;
|
2012-04-17 17:36:46 +02:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return options;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
DBImpl* dbfull() {
|
|
|
|
return reinterpret_cast<DBImpl*>(db_);
|
|
|
|
}
|
|
|
|
|
|
|
|
void Reopen(Options* options = NULL) {
|
|
|
|
ASSERT_OK(TryReopen(options));
|
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
void Close() {
|
|
|
|
delete db_;
|
|
|
|
db_ = NULL;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
void DestroyAndReopen(Options* options = NULL) {
|
|
|
|
delete db_;
|
|
|
|
db_ = NULL;
|
|
|
|
DestroyDB(dbname_, Options());
|
|
|
|
ASSERT_OK(TryReopen(options));
|
|
|
|
}
|
|
|
|
|
2012-08-18 09:26:50 +02:00
|
|
|
Status PureReopen(Options* options, DB** db) {
|
|
|
|
return DB::Open(*options, dbname_, db);
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Status TryReopen(Options* options) {
|
|
|
|
delete db_;
|
|
|
|
db_ = NULL;
|
|
|
|
Options opts;
|
|
|
|
if (options != NULL) {
|
|
|
|
opts = *options;
|
|
|
|
} else {
|
2012-04-17 17:36:46 +02:00
|
|
|
opts = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
opts.create_if_missing = true;
|
|
|
|
}
|
|
|
|
last_options_ = opts;
|
|
|
|
|
|
|
|
return DB::Open(opts, dbname_, &db_);
|
|
|
|
}
|
|
|
|
|
|
|
|
Status Put(const std::string& k, const std::string& v) {
|
2011-04-12 21:38:58 +02:00
|
|
|
return db_->Put(WriteOptions(), k, v);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
Status Delete(const std::string& k) {
|
2011-04-12 21:38:58 +02:00
|
|
|
return db_->Delete(WriteOptions(), k);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
std::string Get(const std::string& k, const Snapshot* snapshot = NULL) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.snapshot = snapshot;
|
|
|
|
std::string result;
|
|
|
|
Status s = db_->Get(options, k, &result);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
result = "NOT_FOUND";
|
|
|
|
} else if (!s.ok()) {
|
|
|
|
result = s.ToString();
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
// Return a string that contains all key,value pairs in order,
|
|
|
|
// formatted like "(k1->v1)(k2->v2)".
|
|
|
|
std::string Contents() {
|
|
|
|
std::vector<std::string> forward;
|
|
|
|
std::string result;
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
|
|
|
std::string s = IterStatus(iter);
|
|
|
|
result.push_back('(');
|
|
|
|
result.append(s);
|
|
|
|
result.push_back(')');
|
|
|
|
forward.push_back(s);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check reverse iteration results are the reverse of forward results
|
2012-11-06 21:02:18 +01:00
|
|
|
unsigned int matched = 0;
|
2011-10-31 18:22:06 +01:00
|
|
|
for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
|
|
|
|
ASSERT_LT(matched, forward.size());
|
|
|
|
ASSERT_EQ(IterStatus(iter), forward[forward.size() - matched - 1]);
|
|
|
|
matched++;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(matched, forward.size());
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
std::string AllEntriesFor(const Slice& user_key) {
|
|
|
|
Iterator* iter = dbfull()->TEST_NewInternalIterator();
|
|
|
|
InternalKey target(user_key, kMaxSequenceNumber, kTypeValue);
|
|
|
|
iter->Seek(target.Encode());
|
|
|
|
std::string result;
|
|
|
|
if (!iter->status().ok()) {
|
|
|
|
result = iter->status().ToString();
|
|
|
|
} else {
|
|
|
|
result = "[ ";
|
|
|
|
bool first = true;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
ParsedInternalKey ikey;
|
|
|
|
if (!ParseInternalKey(iter->key(), &ikey)) {
|
|
|
|
result += "CORRUPTED";
|
|
|
|
} else {
|
2012-04-17 17:36:46 +02:00
|
|
|
if (last_options_.comparator->Compare(ikey.user_key, user_key) != 0) {
|
2011-03-18 23:37:00 +01:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (!first) {
|
|
|
|
result += ", ";
|
|
|
|
}
|
|
|
|
first = false;
|
|
|
|
switch (ikey.type) {
|
|
|
|
case kTypeValue:
|
|
|
|
result += iter->value().ToString();
|
|
|
|
break;
|
|
|
|
case kTypeDeletion:
|
|
|
|
result += "DEL";
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
if (!first) {
|
|
|
|
result += " ";
|
|
|
|
}
|
|
|
|
result += "]";
|
|
|
|
}
|
|
|
|
delete iter;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
int NumTableFilesAtLevel(int level) {
|
2011-04-12 21:38:58 +02:00
|
|
|
std::string property;
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_TRUE(
|
|
|
|
db_->GetProperty("leveldb.num-files-at-level" + NumberToString(level),
|
2011-04-12 21:38:58 +02:00
|
|
|
&property));
|
|
|
|
return atoi(property.c_str());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
int TotalTableFiles() {
|
|
|
|
int result = 0;
|
2012-06-23 04:30:03 +02:00
|
|
|
for (int level = 0; level < db_->NumberLevels(); level++) {
|
2011-06-22 04:36:45 +02:00
|
|
|
result += NumTableFilesAtLevel(level);
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
// Return spread of files per level
|
|
|
|
std::string FilesPerLevel() {
|
|
|
|
std::string result;
|
|
|
|
int last_non_zero_offset = 0;
|
2012-06-23 04:30:03 +02:00
|
|
|
for (int level = 0; level < db_->NumberLevels(); level++) {
|
2011-10-06 01:30:28 +02:00
|
|
|
int f = NumTableFilesAtLevel(level);
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
|
|
|
|
result += buf;
|
|
|
|
if (f > 0) {
|
|
|
|
last_non_zero_offset = result.size();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
result.resize(last_non_zero_offset);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2012-01-25 23:56:52 +01:00
|
|
|
int CountFiles() {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
env_->GetChildren(dbname_, &files);
|
|
|
|
return static_cast<int>(files.size());
|
|
|
|
}
|
|
|
|
|
2012-10-29 19:12:24 +01:00
|
|
|
int CountLiveFiles() {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
uint64_t manifest_file_size;
|
|
|
|
db_->GetLiveFiles(files, &manifest_file_size);
|
|
|
|
return files.size();
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
uint64_t Size(const Slice& start, const Slice& limit) {
|
|
|
|
Range r(start, limit);
|
|
|
|
uint64_t size;
|
|
|
|
db_->GetApproximateSizes(&r, 1, &size);
|
|
|
|
return size;
|
|
|
|
}
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
void Compact(const Slice& start, const Slice& limit) {
|
2011-10-06 01:30:28 +02:00
|
|
|
db_->CompactRange(&start, &limit);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Do n memtable compactions, each of which produces an sstable
|
|
|
|
// covering the range [small,large].
|
|
|
|
void MakeTables(int n, const std::string& small, const std::string& large) {
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
Put(small, "begin");
|
|
|
|
Put(large, "end");
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
2011-03-22 19:32:49 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Prevent pushing of new sstables into deeper levels by adding
|
|
|
|
// tables that cover a specified range to all levels.
|
|
|
|
void FillLevels(const std::string& smallest, const std::string& largest) {
|
2012-06-23 04:30:03 +02:00
|
|
|
MakeTables(db_->NumberLevels(), smallest, largest);
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
void DumpFileCounts(const char* label) {
|
|
|
|
fprintf(stderr, "---\n%s:\n", label);
|
|
|
|
fprintf(stderr, "maxoverlap: %lld\n",
|
|
|
|
static_cast<long long>(
|
|
|
|
dbfull()->TEST_MaxNextLevelOverlappingBytes()));
|
2012-06-23 04:30:03 +02:00
|
|
|
for (int level = 0; level < db_->NumberLevels(); level++) {
|
2011-03-22 19:32:49 +01:00
|
|
|
int num = NumTableFilesAtLevel(level);
|
|
|
|
if (num > 0) {
|
|
|
|
fprintf(stderr, " level %3d : %d files\n", level, num);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-03-25 21:27:43 +01:00
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
std::string DumpSSTableList() {
|
|
|
|
std::string property;
|
|
|
|
db_->GetProperty("leveldb.sstables", &property);
|
|
|
|
return property;
|
|
|
|
}
|
|
|
|
|
2011-03-25 21:27:43 +01:00
|
|
|
std::string IterStatus(Iterator* iter) {
|
|
|
|
std::string result;
|
|
|
|
if (iter->Valid()) {
|
|
|
|
result = iter->key().ToString() + "->" + iter->value().ToString();
|
|
|
|
} else {
|
|
|
|
result = "(invalid)";
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
TEST(DBTest, Empty) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_TRUE(db_ != NULL);
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, ReadWrite) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
ASSERT_OK(Put("bar", "v2"));
|
|
|
|
ASSERT_OK(Put("foo", "v3"));
|
|
|
|
ASSERT_EQ("v3", Get("foo"));
|
|
|
|
ASSERT_EQ("v2", Get("bar"));
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, PutDeleteGet) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
ASSERT_OK(db_->Put(WriteOptions(), "foo", "v2"));
|
|
|
|
ASSERT_EQ("v2", Get("foo"));
|
|
|
|
ASSERT_OK(db_->Delete(WriteOptions(), "foo"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
TEST(DBTest, GetFromImmutableLayer) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000; // Small write buffer
|
|
|
|
Reopen(&options);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
env_->delay_sstable_sync_.Release_Store(env_); // Block sync calls
|
|
|
|
Put("k1", std::string(100000, 'x')); // Fill memtable
|
|
|
|
Put("k2", std::string(100000, 'y')); // Trigger compaction
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
env_->delay_sstable_sync_.Release_Store(NULL); // Release sync calls
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetFromVersions) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetSnapshot) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
// Try with both a short key and a long key
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
|
|
|
|
ASSERT_OK(Put(key, "v1"));
|
|
|
|
const Snapshot* s1 = db_->GetSnapshot();
|
|
|
|
ASSERT_OK(Put(key, "v2"));
|
|
|
|
ASSERT_EQ("v2", Get(key));
|
|
|
|
ASSERT_EQ("v1", Get(key, s1));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("v2", Get(key));
|
|
|
|
ASSERT_EQ("v1", Get(key, s1));
|
|
|
|
db_->ReleaseSnapshot(s1);
|
|
|
|
}
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetLevel0Ordering) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
// Check that we process level-0 files in correct order. The code
|
|
|
|
// below generates two level-0 files where the earlier one comes
|
|
|
|
// before the later one in the level-0 file list since the earlier
|
|
|
|
// one has a smaller "smallest" key.
|
|
|
|
ASSERT_OK(Put("bar", "b"));
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_OK(Put("foo", "v2"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("v2", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetOrderedByLevels) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
Compact("a", "z");
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
ASSERT_OK(Put("foo", "v2"));
|
|
|
|
ASSERT_EQ("v2", Get("foo"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("v2", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, GetPicksCorrectFile) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
// Arrange to have multiple files in a non-level-0 level.
|
|
|
|
ASSERT_OK(Put("a", "va"));
|
|
|
|
Compact("a", "b");
|
|
|
|
ASSERT_OK(Put("x", "vx"));
|
|
|
|
Compact("x", "y");
|
|
|
|
ASSERT_OK(Put("f", "vf"));
|
|
|
|
Compact("f", "g");
|
|
|
|
ASSERT_EQ("va", Get("a"));
|
|
|
|
ASSERT_EQ("vf", Get("f"));
|
|
|
|
ASSERT_EQ("vx", Get("x"));
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-09-01 21:08:02 +02:00
|
|
|
TEST(DBTest, GetEncountersEmptyLevel) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
// Arrange for the following to happen:
|
|
|
|
// * sstable A in level 0
|
|
|
|
// * nothing in level 1
|
|
|
|
// * sstable B in level 2
|
|
|
|
// Then do enough Get() calls to arrange for an automatic compaction
|
|
|
|
// of sstable A. A bug would cause the compaction to be marked as
|
|
|
|
// occuring at level 1 (instead of the correct level 0).
|
|
|
|
|
|
|
|
// Step 1: First place sstables in levels 0 and 2
|
|
|
|
int compaction_count = 0;
|
|
|
|
while (NumTableFilesAtLevel(0) == 0 ||
|
|
|
|
NumTableFilesAtLevel(2) == 0) {
|
|
|
|
ASSERT_LE(compaction_count, 100) << "could not fill levels 0 and 2";
|
|
|
|
compaction_count++;
|
|
|
|
Put("a", "begin");
|
|
|
|
Put("z", "end");
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
}
|
2011-09-01 21:08:02 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Step 2: clear level 1 if necessary.
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(2), 1);
|
|
|
|
|
2012-08-27 08:45:35 +02:00
|
|
|
// Step 3: read a bunch of times
|
|
|
|
for (int i = 0; i < 1000; i++) {
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_EQ("NOT_FOUND", Get("missing"));
|
|
|
|
}
|
2012-08-27 08:45:35 +02:00
|
|
|
|
|
|
|
// Step 4: Wait for compaction to finish
|
|
|
|
env_->SleepForMicroseconds(1000000);
|
|
|
|
|
2012-09-27 10:05:38 +02:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 1); // XXX
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-09-01 21:08:02 +02:00
|
|
|
}
|
|
|
|
|
2011-03-25 21:27:43 +01:00
|
|
|
TEST(DBTest, IterEmpty) {
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("foo");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterSingle) {
|
|
|
|
ASSERT_OK(Put("a", "va"));
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("a");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("b");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterMulti) {
|
|
|
|
ASSERT_OK(Put("a", "va"));
|
|
|
|
ASSERT_OK(Put("b", "vb"));
|
|
|
|
ASSERT_OK(Put("c", "vc"));
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->Seek("");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Seek("a");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Seek("ax");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Seek("b");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Seek("z");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
// Switch from reverse to forward
|
|
|
|
iter->SeekToLast();
|
|
|
|
iter->Prev();
|
|
|
|
iter->Prev();
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
|
|
|
|
// Switch from forward to reverse
|
|
|
|
iter->SeekToFirst();
|
|
|
|
iter->Next();
|
|
|
|
iter->Next();
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
|
|
|
|
// Make sure iter stays at snapshot
|
|
|
|
ASSERT_OK(Put("a", "va2"));
|
|
|
|
ASSERT_OK(Put("a2", "va3"));
|
|
|
|
ASSERT_OK(Put("b", "vb2"));
|
|
|
|
ASSERT_OK(Put("c", "vc2"));
|
|
|
|
ASSERT_OK(Delete("b"));
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->vb");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IterSmallAndLargeMix) {
|
|
|
|
ASSERT_OK(Put("a", "va"));
|
|
|
|
ASSERT_OK(Put("b", std::string(100000, 'b')));
|
|
|
|
ASSERT_OK(Put("c", "vc"));
|
|
|
|
ASSERT_OK(Put("d", std::string(100000, 'd')));
|
|
|
|
ASSERT_OK(Put("e", std::string(100000, 'e')));
|
|
|
|
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
iter->SeekToLast();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "e->" + std::string(100000, 'e'));
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "d->" + std::string(100000, 'd'));
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "b->" + std::string(100000, 'b'));
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "(invalid)");
|
|
|
|
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
2011-08-16 03:21:01 +02:00
|
|
|
TEST(DBTest, IterMultiWithDelete) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("a", "va"));
|
|
|
|
ASSERT_OK(Put("b", "vb"));
|
|
|
|
ASSERT_OK(Put("c", "vc"));
|
|
|
|
ASSERT_OK(Delete("b"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get("b"));
|
2011-08-16 03:21:01 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
iter->Seek("c");
|
|
|
|
ASSERT_EQ(IterStatus(iter), "c->vc");
|
|
|
|
iter->Prev();
|
|
|
|
ASSERT_EQ(IterStatus(iter), "a->va");
|
|
|
|
delete iter;
|
|
|
|
} while (ChangeOptions());
|
2011-08-16 03:21:01 +02:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, Recover) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_OK(Put("baz", "v5"));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
ASSERT_EQ("v5", Get("baz"));
|
|
|
|
ASSERT_OK(Put("bar", "v2"));
|
|
|
|
ASSERT_OK(Put("foo", "v3"));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("v3", Get("foo"));
|
|
|
|
ASSERT_OK(Put("foo", "v4"));
|
|
|
|
ASSERT_EQ("v4", Get("foo"));
|
|
|
|
ASSERT_EQ("v2", Get("bar"));
|
|
|
|
ASSERT_EQ("v5", Get("baz"));
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-08-18 01:06:05 +02:00
|
|
|
TEST(DBTest, RollLog) {
|
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_OK(Put("baz", "v5"));
|
|
|
|
|
|
|
|
Reopen();
|
|
|
|
for (int i = 0; i < 10; i++) {
|
|
|
|
Reopen();
|
|
|
|
}
|
|
|
|
ASSERT_OK(Put("foo", "v4"));
|
|
|
|
for (int i = 0; i < 10; i++) {
|
|
|
|
Reopen();
|
|
|
|
}
|
|
|
|
} while (ChangeOptions());
|
|
|
|
}
|
|
|
|
|
2012-07-05 22:39:28 +02:00
|
|
|
TEST(DBTest, WAL) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v1"));
|
2012-07-06 20:42:09 +02:00
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v1"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
|
|
|
Reopen();
|
2012-11-06 20:21:57 +01:00
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
ASSERT_EQ("v1", Get("bar"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
|
|
|
writeOpt.disableWAL = false;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v2"));
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v2"));
|
|
|
|
|
|
|
|
Reopen();
|
2012-11-06 20:21:57 +01:00
|
|
|
// Both value's should be present.
|
2012-07-05 22:39:28 +02:00
|
|
|
ASSERT_EQ("v2", Get("bar"));
|
2012-11-06 20:21:57 +01:00
|
|
|
ASSERT_EQ("v2", Get("foo"));
|
2012-07-05 22:39:28 +02:00
|
|
|
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v3"));
|
|
|
|
writeOpt.disableWAL = false;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v3"));
|
|
|
|
|
|
|
|
Reopen();
|
2012-11-06 20:21:57 +01:00
|
|
|
// again both values should be present.
|
2012-07-05 22:39:28 +02:00
|
|
|
ASSERT_EQ("v3", Get("foo"));
|
2012-11-06 20:21:57 +01:00
|
|
|
ASSERT_EQ("v3", Get("bar"));
|
2012-07-05 22:39:28 +02:00
|
|
|
}
|
|
|
|
|
2012-08-18 09:26:50 +02:00
|
|
|
TEST(DBTest, CheckLock) {
|
|
|
|
DB* localdb;
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
ASSERT_TRUE(TryReopen(&options).ok());
|
|
|
|
ASSERT_TRUE(!(PureReopen(&options, &localdb).ok())); // second open should fail
|
|
|
|
}
|
|
|
|
|
2012-07-06 20:42:09 +02:00
|
|
|
TEST(DBTest, FLUSH) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
WriteOptions writeOpt = WriteOptions();
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v1"));
|
2012-11-06 20:21:57 +01:00
|
|
|
// this will now also flush the last 2 writes
|
2012-07-06 20:42:09 +02:00
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v1"));
|
|
|
|
|
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
2012-11-06 20:21:57 +01:00
|
|
|
ASSERT_EQ("v1", Get("bar"));
|
2012-07-06 20:42:09 +02:00
|
|
|
|
|
|
|
writeOpt.disableWAL = true;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v2"));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v2"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
|
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("v2", Get("bar"));
|
|
|
|
ASSERT_EQ("v2", Get("foo"));
|
|
|
|
|
|
|
|
writeOpt.disableWAL = false;
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v3"));
|
|
|
|
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v3"));
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
|
|
|
|
Reopen();
|
|
|
|
// 'foo' should be there because its put
|
|
|
|
// has WAL enabled.
|
|
|
|
ASSERT_EQ("v3", Get("foo"));
|
|
|
|
ASSERT_EQ("v3", Get("bar"));
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, RecoveryWithEmptyLog) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_OK(Put("foo", "v2"));
|
|
|
|
Reopen();
|
|
|
|
Reopen();
|
|
|
|
ASSERT_OK(Put("foo", "v3"));
|
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("v3", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
// Check that writes done during a memtable compaction are recovered
|
|
|
|
// if the database is shutdown during the memtable compaction.
|
|
|
|
TEST(DBTest, RecoverDuringMemtableCompaction) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 1000000;
|
|
|
|
Reopen(&options);
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Trigger a long memtable compaction and reopen the database during it
|
|
|
|
ASSERT_OK(Put("foo", "v1")); // Goes to 1st log file
|
|
|
|
ASSERT_OK(Put("big1", std::string(10000000, 'x'))); // Fills memtable
|
|
|
|
ASSERT_OK(Put("big2", std::string(1000, 'y'))); // Triggers compaction
|
|
|
|
ASSERT_OK(Put("bar", "v2")); // Goes to new log file
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
ASSERT_EQ("v2", Get("bar"));
|
|
|
|
ASSERT_EQ(std::string(10000000, 'x'), Get("big1"));
|
|
|
|
ASSERT_EQ(std::string(1000, 'y'), Get("big2"));
|
|
|
|
} while (ChangeOptions());
|
2011-06-22 04:36:45 +02:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
static std::string Key(int i) {
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "key%06d", i);
|
|
|
|
return std::string(buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, MinorCompactionsHappen) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
options.write_buffer_size = 10000;
|
|
|
|
Reopen(&options);
|
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
const int N = 500;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
int starting_num_tables = TotalTableFiles();
|
2011-03-18 23:37:00 +01:00
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_OK(Put(Key(i), Key(i) + std::string(1000, 'v')));
|
|
|
|
}
|
2011-06-22 04:36:45 +02:00
|
|
|
int ending_num_tables = TotalTableFiles();
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_GT(ending_num_tables, starting_num_tables);
|
|
|
|
|
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
|
|
|
|
}
|
|
|
|
|
|
|
|
Reopen();
|
|
|
|
|
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_EQ(Key(i) + std::string(1000, 'v'), Get(Key(i)));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, RecoverWithLargeLog) {
|
|
|
|
{
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_OK(Put("big1", std::string(200000, '1')));
|
|
|
|
ASSERT_OK(Put("big2", std::string(200000, '2')));
|
|
|
|
ASSERT_OK(Put("small3", std::string(10, '3')));
|
|
|
|
ASSERT_OK(Put("small4", std::string(10, '4')));
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Make sure that if we re-open with a small write buffer size that
|
|
|
|
// we flush table files in the middle of a large log file.
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
options.write_buffer_size = 100000;
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 3);
|
|
|
|
ASSERT_EQ(std::string(200000, '1'), Get("big1"));
|
|
|
|
ASSERT_EQ(std::string(200000, '2'), Get("big2"));
|
|
|
|
ASSERT_EQ(std::string(10, '3'), Get("small3"));
|
|
|
|
ASSERT_EQ(std::string(10, '4'), Get("small4"));
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(0), 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionsGenerateMultipleFiles) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
options.write_buffer_size = 100000000; // Large write buffer
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Write 8MB (80 values, each 100K)
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 80; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 100000));
|
|
|
|
ASSERT_OK(Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reopening moves updates to level-0
|
|
|
|
Reopen(&options);
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(1), 1);
|
|
|
|
for (int i = 0; i < 80; i++) {
|
|
|
|
ASSERT_EQ(Get(Key(i)), values[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-06-23 04:30:03 +02:00
|
|
|
TEST(DBTest, CompactionTrigger) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
for (int num = 0;
|
2012-11-27 06:16:21 +01:00
|
|
|
num < options.level0_file_num_compaction_trigger - 1;
|
|
|
|
num++) {
|
2012-06-23 04:30:03 +02:00
|
|
|
std::vector<std::string> values;
|
|
|
|
// Write 120KB (12 values, each 10K)
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
|
|
|
ASSERT_OK(Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompactMemTable();
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
//generate one more file in level-0, and should trigger level-0 compaction
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
|
|
|
ASSERT_OK(Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 1);
|
|
|
|
}
|
|
|
|
|
2012-11-01 18:50:08 +01:00
|
|
|
void MinLevelHelper(DBTest* self, Options& options) {
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
for (int num = 0;
|
|
|
|
num < options.level0_file_num_compaction_trigger - 1;
|
|
|
|
num++)
|
|
|
|
{
|
|
|
|
std::vector<std::string> values;
|
|
|
|
// Write 120KB (12 values, each 10K)
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
|
|
|
ASSERT_OK(self->Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
self->dbfull()->TEST_WaitForCompactMemTable();
|
|
|
|
ASSERT_EQ(self->NumTableFilesAtLevel(0), num + 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
//generate one more file in level-0, and should trigger level-0 compaction
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 10000));
|
|
|
|
ASSERT_OK(self->Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
self->dbfull()->TEST_WaitForCompact();
|
|
|
|
|
|
|
|
ASSERT_EQ(self->NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(self->NumTableFilesAtLevel(1), 1);
|
|
|
|
}
|
|
|
|
|
2012-11-16 21:55:21 +01:00
|
|
|
// returns false if the calling-Test should be skipped
|
|
|
|
bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
|
2012-11-01 18:50:08 +01:00
|
|
|
int lev, int strategy) {
|
2012-11-05 07:04:14 +01:00
|
|
|
fprintf(stderr, "Test with compression options : window_bits = %d, level = %d, strategy = %d}\n", wbits, lev, strategy);
|
2012-11-01 18:50:08 +01:00
|
|
|
options.write_buffer_size = 100<<10; //100KB
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
options.level0_file_num_compaction_trigger = 3;
|
2012-10-28 07:13:17 +01:00
|
|
|
options.create_if_missing = true;
|
2012-11-01 18:50:08 +01:00
|
|
|
|
|
|
|
if (SnappyCompressionSupported(CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kSnappyCompression;
|
|
|
|
fprintf(stderr, "using snappy\n");
|
|
|
|
} else if (ZlibCompressionSupported(
|
|
|
|
CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kZlibCompression;
|
|
|
|
fprintf(stderr, "using zlib\n");
|
|
|
|
} else if (BZip2CompressionSupported(
|
|
|
|
CompressionOptions(wbits, lev, strategy))) {
|
|
|
|
type = kBZip2Compression;
|
|
|
|
fprintf(stderr, "using bzip2\n");
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "skipping test, compression disabled\n");
|
2012-11-16 21:55:21 +01:00
|
|
|
return false;
|
2012-11-01 18:50:08 +01:00
|
|
|
}
|
2012-10-28 07:13:17 +01:00
|
|
|
options.compression_per_level = new CompressionType[options.num_levels];
|
|
|
|
|
|
|
|
// do not compress L0
|
|
|
|
for (int i = 0; i < 1; i++) {
|
|
|
|
options.compression_per_level[i] = kNoCompression;
|
|
|
|
}
|
|
|
|
for (int i = 1; i < options.num_levels; i++) {
|
|
|
|
options.compression_per_level[i] = type;
|
|
|
|
}
|
2012-11-16 21:55:21 +01:00
|
|
|
return true;
|
2012-11-01 18:50:08 +01:00
|
|
|
}
|
|
|
|
TEST(DBTest, MinLevelToCompress1) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
CompressionType type;
|
2012-11-16 21:55:21 +01:00
|
|
|
if (!MinLevelToCompress(type, options, -14, -1, 0)) {
|
|
|
|
return;
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
Reopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
|
|
|
|
// do not compress L0 and L1
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
options.compression_per_level[i] = kNoCompression;
|
|
|
|
}
|
|
|
|
for (int i = 2; i < options.num_levels; i++) {
|
|
|
|
options.compression_per_level[i] = type;
|
|
|
|
}
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, MinLevelToCompress2) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
CompressionType type;
|
2012-11-16 21:55:21 +01:00
|
|
|
if (!MinLevelToCompress(type, options, 15, -1, 0)) {
|
|
|
|
return;
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
Reopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
|
2012-10-28 07:13:17 +01:00
|
|
|
// do not compress L0 and L1
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
options.compression_per_level[i] = kNoCompression;
|
|
|
|
}
|
|
|
|
for (int i = 2; i < options.num_levels; i++) {
|
|
|
|
options.compression_per_level[i] = type;
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
DestroyAndReopen(&options);
|
|
|
|
MinLevelHelper(this, options);
|
|
|
|
}
|
2012-10-28 07:13:17 +01:00
|
|
|
|
2011-07-15 02:20:57 +02:00
|
|
|
TEST(DBTest, RepeatedWritesToSameKey) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2011-07-15 02:20:57 +02:00
|
|
|
options.env = env_;
|
|
|
|
options.write_buffer_size = 100000; // Small write buffer
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
// We must have at most one file per level except for level-0,
|
|
|
|
// which may have up to kL0_StopWritesTrigger files.
|
2012-06-23 04:30:03 +02:00
|
|
|
const int kMaxFiles = dbfull()->NumberLevels() +
|
2012-11-26 22:56:45 +01:00
|
|
|
dbfull()->Level0StopWriteTrigger();
|
2011-07-15 02:20:57 +02:00
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
|
|
|
|
for (int i = 0; i < 5 * kMaxFiles; i++) {
|
|
|
|
Put("key", value);
|
|
|
|
ASSERT_LE(TotalTableFiles(), kMaxFiles);
|
|
|
|
fprintf(stderr, "after %d: %d files\n", int(i+1), TotalTableFiles());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-10-29 09:13:41 +01:00
|
|
|
// This is a static filter used for filtering
|
|
|
|
// kvs during the compaction process.
|
|
|
|
static int cfilter_count;
|
|
|
|
static std::string NEW_VALUE = "NewValue";
|
2012-11-14 00:48:00 +01:00
|
|
|
static bool keep_filter(void* arg, int level, const Slice& key,
|
2012-10-29 09:13:41 +01:00
|
|
|
const Slice& value, Slice** new_value) {
|
2012-11-14 00:48:00 +01:00
|
|
|
assert(arg == NULL);
|
2012-11-06 20:21:57 +01:00
|
|
|
cfilter_count++;
|
2012-10-29 09:13:41 +01:00
|
|
|
return false;
|
|
|
|
}
|
2012-11-14 00:48:00 +01:00
|
|
|
static bool delete_filter(void*argv, int level, const Slice& key,
|
2012-10-29 09:13:41 +01:00
|
|
|
const Slice& value, Slice** new_value) {
|
2012-11-19 22:16:46 +01:00
|
|
|
assert(argv == NULL);
|
2012-11-06 20:21:57 +01:00
|
|
|
cfilter_count++;
|
2012-10-29 09:13:41 +01:00
|
|
|
return true;
|
|
|
|
}
|
2012-11-14 00:48:00 +01:00
|
|
|
static bool change_filter(void*argv, int level, const Slice& key,
|
2012-10-29 09:13:41 +01:00
|
|
|
const Slice& value, Slice** new_value) {
|
2012-11-14 00:48:00 +01:00
|
|
|
assert(argv == (void*)100);
|
2012-10-29 09:13:41 +01:00
|
|
|
assert(new_value != NULL);
|
|
|
|
*new_value = new Slice(NEW_VALUE);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionFilter) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
|
|
|
options.CompactionFilter = keep_filter;
|
|
|
|
Reopen(&options);
|
|
|
|
|
2012-11-06 20:21:57 +01:00
|
|
|
// Write 100K+1 keys, these are written to a few files
|
|
|
|
// in L0. We do this so that the current snapshot points
|
2012-10-29 09:13:41 +01:00
|
|
|
// to the 100001 key.The compaction filter is not invoked
|
|
|
|
// on keys that are visible via a snapshot because we
|
|
|
|
// anyways cannot delete it.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
|
|
|
|
// Push all files to the highest level L2. Verify that
|
|
|
|
// the compaction is each level invokes the filter for
|
|
|
|
// all the keys in that level.
|
|
|
|
cfilter_count = 0;
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
cfilter_count = 0;
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(2), 0);
|
|
|
|
cfilter_count = 0;
|
|
|
|
|
|
|
|
// overwrite all the 100K+1 keys once again.
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
|
|
|
|
// push all files to the highest level L2. This
|
|
|
|
// means that all keys should pass at least once
|
|
|
|
// via the compaction filter
|
|
|
|
cfilter_count = 0;
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
cfilter_count = 0;
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(2), 0);
|
|
|
|
|
|
|
|
// create a new database with the compaction
|
|
|
|
// filter in such a way that it deletes all keys
|
|
|
|
options.CompactionFilter = delete_filter;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
// write all the keys once again.
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(2), 0);
|
|
|
|
|
|
|
|
// Push all files to the highest level L2. This
|
|
|
|
// triggers the compaction filter to delete all keys,
|
|
|
|
// verify that at the end of the compaction process,
|
|
|
|
// nothing is left.
|
|
|
|
cfilter_count = 0;
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
ASSERT_EQ(cfilter_count, 100000);
|
|
|
|
cfilter_count = 0;
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
ASSERT_EQ(cfilter_count, 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
|
|
|
|
// Scan the entire database to ensure that only the
|
|
|
|
// 100001th key is left in the db. The 100001th key
|
|
|
|
// is part of the default-most-current snapshot and
|
|
|
|
// cannot be deleted.
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
iter->SeekToFirst();
|
|
|
|
int count = 0;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
count++;
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
ASSERT_EQ(count, 1);
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, CompactionFilterWithValueChange) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.num_levels = 3;
|
|
|
|
options.max_mem_compaction_level = 0;
|
2012-11-14 00:48:00 +01:00
|
|
|
options.compaction_filter_args = (void *)100;
|
2012-10-29 09:13:41 +01:00
|
|
|
options.CompactionFilter = change_filter;
|
|
|
|
Reopen(&options);
|
|
|
|
|
2012-11-06 20:21:57 +01:00
|
|
|
// Write 100K+1 keys, these are written to a few files
|
|
|
|
// in L0. We do this so that the current snapshot points
|
2012-10-29 09:13:41 +01:00
|
|
|
// to the 100001 key.The compaction filter is not invoked
|
|
|
|
// on keys that are visible via a snapshot because we
|
|
|
|
// anyways cannot delete it.
|
|
|
|
const std::string value(10, 'x');
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
|
|
|
|
// push all files to lower levels
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
|
|
|
|
// re-write all data again
|
|
|
|
for (int i = 0; i < 100001; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
|
|
|
|
// push all files to lower levels. This should
|
|
|
|
// invoke the compaction filter for all 100000 keys.
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
|
|
|
|
// verify that all keys now have the new value that
|
|
|
|
// was set by the compaction process.
|
|
|
|
for (int i = 0; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
std::string newvalue = Get(key);
|
|
|
|
ASSERT_EQ(newvalue.compare(NEW_VALUE), 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
TEST(DBTest, SparseMerge) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2011-03-22 19:32:49 +01:00
|
|
|
options.compression = kNoCompression;
|
|
|
|
Reopen(&options);
|
|
|
|
|
2011-06-22 04:36:45 +02:00
|
|
|
FillLevels("A", "Z");
|
|
|
|
|
2011-03-22 19:32:49 +01:00
|
|
|
// Suppose there is:
|
|
|
|
// small amount of data with prefix A
|
|
|
|
// large amount of data with prefix B
|
|
|
|
// small amount of data with prefix C
|
|
|
|
// and that recent updates have made small changes to all three prefixes.
|
|
|
|
// Check that we do not do a compaction that merges all of B in one shot.
|
|
|
|
const std::string value(1000, 'x');
|
|
|
|
Put("A", "va");
|
|
|
|
// Write approximately 100MB of "B" values
|
|
|
|
for (int i = 0; i < 100000; i++) {
|
|
|
|
char key[100];
|
|
|
|
snprintf(key, sizeof(key), "B%010d", i);
|
|
|
|
Put(key, value);
|
|
|
|
}
|
|
|
|
Put("C", "vc");
|
2011-06-22 04:36:45 +02:00
|
|
|
dbfull()->TEST_CompactMemTable();
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
2011-03-22 19:32:49 +01:00
|
|
|
|
|
|
|
// Make sparse update
|
|
|
|
Put("A", "va2");
|
|
|
|
Put("B100", "bvalue2");
|
|
|
|
Put("C", "vc2");
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
|
|
|
|
// Compactions should not cause us to create a situation where
|
|
|
|
// a file overlaps too much data at the next level.
|
|
|
|
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
2011-03-22 19:32:49 +01:00
|
|
|
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
2011-03-22 19:32:49 +01:00
|
|
|
ASSERT_LE(dbfull()->TEST_MaxNextLevelOverlappingBytes(), 20*1048576);
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
static bool Between(uint64_t val, uint64_t low, uint64_t high) {
|
|
|
|
bool result = (val >= low) && (val <= high);
|
|
|
|
if (!result) {
|
|
|
|
fprintf(stderr, "Value %llu is not in range [%llu, %llu]\n",
|
|
|
|
(unsigned long long)(val),
|
|
|
|
(unsigned long long)(low),
|
|
|
|
(unsigned long long)(high));
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, ApproximateSizes) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 100000000; // Large write buffer
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
DestroyAndReopen();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_TRUE(Between(Size("", "xyz"), 0, 0));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Write 8MB (80 values, each 100K)
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
const int N = 80;
|
|
|
|
static const int S1 = 100000;
|
|
|
|
static const int S2 = 105000; // Allow some expansion from metadata
|
|
|
|
Random rnd(301);
|
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_OK(Put(Key(i), RandomString(&rnd, S1)));
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// 0 because GetApproximateSizes() does not account for memtable space
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(50)), 0, 0));
|
2011-04-20 01:01:25 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Check sizes across recovery by reopening a few times
|
|
|
|
for (int run = 0; run < 3; run++) {
|
|
|
|
Reopen(&options);
|
2011-04-21 00:48:11 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
for (int compact_start = 0; compact_start < N; compact_start += 10) {
|
|
|
|
for (int i = 0; i < N; i += 10) {
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(i)), S1*i, S2*i));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(i)+".suffix"), S1*(i+1), S2*(i+1)));
|
|
|
|
ASSERT_TRUE(Between(Size(Key(i), Key(i+10)), S1*10, S2*10));
|
|
|
|
}
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(50)), S1*50, S2*50));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(50)+".suffix"), S1*50, S2*50));
|
|
|
|
|
|
|
|
std::string cstart_str = Key(compact_start);
|
|
|
|
std::string cend_str = Key(compact_start + 9);
|
|
|
|
Slice cstart = cstart_str;
|
|
|
|
Slice cend = cend_str;
|
|
|
|
dbfull()->TEST_CompactRange(0, &cstart, &cend);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-04-21 00:48:11 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(1), 0);
|
|
|
|
}
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
Reopen();
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Random rnd(301);
|
|
|
|
std::string big1 = RandomString(&rnd, 100000);
|
|
|
|
ASSERT_OK(Put(Key(0), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(Key(1), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(Key(2), big1));
|
|
|
|
ASSERT_OK(Put(Key(3), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(Key(4), big1));
|
|
|
|
ASSERT_OK(Put(Key(5), RandomString(&rnd, 10000)));
|
|
|
|
ASSERT_OK(Put(Key(6), RandomString(&rnd, 300000)));
|
|
|
|
ASSERT_OK(Put(Key(7), RandomString(&rnd, 10000)));
|
|
|
|
|
|
|
|
// Check sizes across recovery by reopening a few times
|
|
|
|
for (int run = 0; run < 3; run++) {
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(0)), 0, 0));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(1)), 10000, 11000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(2)), 20000, 21000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(3)), 120000, 121000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(4)), 130000, 131000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(5)), 230000, 231000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(6)), 240000, 241000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(7)), 540000, 541000));
|
|
|
|
ASSERT_TRUE(Between(Size("", Key(8)), 550000, 560000));
|
|
|
|
|
|
|
|
ASSERT_TRUE(Between(Size(Key(3), Key(5)), 110000, 111000));
|
|
|
|
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
}
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, IteratorPinsRef) {
|
|
|
|
Put("foo", "hello");
|
|
|
|
|
|
|
|
// Get iterator that will yield the current contents of the DB.
|
|
|
|
Iterator* iter = db_->NewIterator(ReadOptions());
|
|
|
|
|
|
|
|
// Write to force compactions
|
|
|
|
Put("foo", "newvalue1");
|
|
|
|
for (int i = 0; i < 100; i++) {
|
|
|
|
ASSERT_OK(Put(Key(i), Key(i) + std::string(100000, 'v'))); // 100K values
|
|
|
|
}
|
|
|
|
Put("foo", "newvalue2");
|
|
|
|
|
|
|
|
iter->SeekToFirst();
|
|
|
|
ASSERT_TRUE(iter->Valid());
|
|
|
|
ASSERT_EQ("foo", iter->key().ToString());
|
|
|
|
ASSERT_EQ("hello", iter->value().ToString());
|
|
|
|
iter->Next();
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
delete iter;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, Snapshot) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
Put("foo", "v1");
|
|
|
|
const Snapshot* s1 = db_->GetSnapshot();
|
|
|
|
Put("foo", "v2");
|
|
|
|
const Snapshot* s2 = db_->GetSnapshot();
|
|
|
|
Put("foo", "v3");
|
|
|
|
const Snapshot* s3 = db_->GetSnapshot();
|
|
|
|
|
|
|
|
Put("foo", "v4");
|
|
|
|
ASSERT_EQ("v1", Get("foo", s1));
|
|
|
|
ASSERT_EQ("v2", Get("foo", s2));
|
|
|
|
ASSERT_EQ("v3", Get("foo", s3));
|
|
|
|
ASSERT_EQ("v4", Get("foo"));
|
|
|
|
|
|
|
|
db_->ReleaseSnapshot(s3);
|
|
|
|
ASSERT_EQ("v1", Get("foo", s1));
|
|
|
|
ASSERT_EQ("v2", Get("foo", s2));
|
|
|
|
ASSERT_EQ("v4", Get("foo"));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
db_->ReleaseSnapshot(s1);
|
|
|
|
ASSERT_EQ("v2", Get("foo", s2));
|
|
|
|
ASSERT_EQ("v4", Get("foo"));
|
2011-06-22 04:36:45 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
db_->ReleaseSnapshot(s2);
|
|
|
|
ASSERT_EQ("v4", Get("foo"));
|
|
|
|
} while (ChangeOptions());
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
TEST(DBTest, HiddenValuesAreRemoved) {
|
|
|
|
do {
|
|
|
|
Random rnd(301);
|
|
|
|
FillLevels("a", "z");
|
|
|
|
|
|
|
|
std::string big = RandomString(&rnd, 50000);
|
|
|
|
Put("foo", big);
|
|
|
|
Put("pastfoo", "v");
|
|
|
|
const Snapshot* snapshot = db_->GetSnapshot();
|
|
|
|
Put("foo", "tiny");
|
|
|
|
Put("pastfoo2", "v2"); // Advance sequence number one more
|
|
|
|
|
|
|
|
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
|
|
|
ASSERT_GT(NumTableFilesAtLevel(0), 0);
|
|
|
|
|
|
|
|
ASSERT_EQ(big, Get("foo", snapshot));
|
|
|
|
ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
|
|
|
|
db_->ReleaseSnapshot(snapshot);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
|
|
|
|
Slice x("x");
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, &x);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_GE(NumTableFilesAtLevel(1), 1);
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, &x);
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_TRUE(Between(Size("", "pastfoo"), 0, 1000));
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-11-27 06:16:21 +01:00
|
|
|
TEST(DBTest, CompactBetweenSnapshots) {
|
|
|
|
do {
|
|
|
|
Random rnd(301);
|
|
|
|
FillLevels("a", "z");
|
|
|
|
|
|
|
|
Put("foo", "first");
|
|
|
|
const Snapshot* snapshot1 = db_->GetSnapshot();
|
|
|
|
Put("foo", "second");
|
|
|
|
Put("foo", "third");
|
|
|
|
Put("foo", "fourth");
|
|
|
|
const Snapshot* snapshot2 = db_->GetSnapshot();
|
|
|
|
Put("foo", "fifth");
|
|
|
|
Put("foo", "sixth");
|
|
|
|
|
|
|
|
// All entries (including duplicates) exist
|
|
|
|
// before any compaction is triggered.
|
|
|
|
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
|
|
|
ASSERT_EQ("sixth", Get("foo"));
|
|
|
|
ASSERT_EQ("fourth", Get("foo", snapshot2));
|
|
|
|
ASSERT_EQ("first", Get("foo", snapshot1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"),
|
|
|
|
"[ sixth, fifth, fourth, third, second, first ]");
|
|
|
|
|
|
|
|
// After a compaction, "second", "third" and "fifth" should
|
|
|
|
// be removed
|
|
|
|
FillLevels("a", "z");
|
|
|
|
dbfull()->CompactRange(NULL, NULL);
|
|
|
|
ASSERT_EQ("sixth", Get("foo"));
|
|
|
|
ASSERT_EQ("fourth", Get("foo", snapshot2));
|
|
|
|
ASSERT_EQ("first", Get("foo", snapshot1));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ sixth, fourth, first ]");
|
|
|
|
|
|
|
|
// after we release the snapshot1, only two values left
|
|
|
|
db_->ReleaseSnapshot(snapshot1);
|
|
|
|
FillLevels("a", "z");
|
|
|
|
dbfull()->CompactRange(NULL, NULL);
|
|
|
|
|
|
|
|
// We have only one valid snapshot snapshot2. Since snapshot1 is
|
|
|
|
// not valid anymore, "first" should be removed by a compaction.
|
|
|
|
ASSERT_EQ("sixth", Get("foo"));
|
|
|
|
ASSERT_EQ("fourth", Get("foo", snapshot2));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ sixth, fourth ]");
|
|
|
|
|
|
|
|
// after we release the snapshot2, only one value should be left
|
|
|
|
db_->ReleaseSnapshot(snapshot2);
|
|
|
|
FillLevels("a", "z");
|
|
|
|
dbfull()->CompactRange(NULL, NULL);
|
|
|
|
ASSERT_EQ("sixth", Get("foo"));
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ sixth ]");
|
|
|
|
|
|
|
|
} while (ChangeOptions());
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, DeletionMarkers1) {
|
|
|
|
Put("foo", "v1");
|
|
|
|
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
2012-06-23 04:30:03 +02:00
|
|
|
const int last = dbfull()->MaxMemCompactionLevel();
|
2011-06-22 04:36:45 +02:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
|
|
|
|
|
|
|
|
// Place a table at level last-1 to prevent merging with preceding mutation
|
|
|
|
Put("a", "begin");
|
|
|
|
Put("z", "end");
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last), 1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Delete("foo");
|
|
|
|
Put("foo", "v2");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
|
2011-06-22 04:36:45 +02:00
|
|
|
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
|
2011-10-06 01:30:28 +02:00
|
|
|
Slice z("z");
|
|
|
|
dbfull()->TEST_CompactRange(last-2, NULL, &z);
|
2011-03-18 23:37:00 +01:00
|
|
|
// DEL eliminated, but v1 remains because we aren't compacting that level
|
|
|
|
// (DEL can be eliminated because v2 hides v1).
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(last-1, NULL, NULL);
|
2011-06-22 04:36:45 +02:00
|
|
|
// Merging last-1 w/ last, so we are the base level for "foo", so
|
|
|
|
// DEL is removed. (as is v1).
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, DeletionMarkers2) {
|
|
|
|
Put("foo", "v1");
|
|
|
|
ASSERT_OK(dbfull()->TEST_CompactMemTable());
|
2012-06-23 04:30:03 +02:00
|
|
|
const int last = dbfull()->MaxMemCompactionLevel();
|
2011-06-22 04:36:45 +02:00
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
|
|
|
|
|
|
|
|
// Place a table at level last-1 to prevent merging with preceding mutation
|
|
|
|
Put("a", "begin");
|
|
|
|
Put("z", "end");
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last), 1);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
Delete("foo");
|
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
2011-06-22 04:36:45 +02:00
|
|
|
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(last-2, NULL, NULL);
|
2011-06-22 04:36:45 +02:00
|
|
|
// DEL kept: "last" file overlaps
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
|
2011-10-06 01:30:28 +02:00
|
|
|
dbfull()->TEST_CompactRange(last-1, NULL, NULL);
|
2011-06-22 04:36:45 +02:00
|
|
|
// Merging last-1 w/ last, so we are the base level for "foo", so
|
|
|
|
// DEL is removed. (as is v1).
|
2011-03-18 23:37:00 +01:00
|
|
|
ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
|
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
TEST(DBTest, OverlapInLevel0) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
2012-11-26 22:56:45 +01:00
|
|
|
int tmp = dbfull()->MaxMemCompactionLevel();
|
2012-06-23 04:30:03 +02:00
|
|
|
ASSERT_EQ(tmp, 2) << "Fix test to match config";
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
|
|
|
|
ASSERT_OK(Put("100", "v100"));
|
|
|
|
ASSERT_OK(Put("999", "v999"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_OK(Delete("100"));
|
|
|
|
ASSERT_OK(Delete("999"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("0,1,1", FilesPerLevel());
|
|
|
|
|
|
|
|
// Make files spanning the following ranges in level-0:
|
|
|
|
// files[0] 200 .. 900
|
|
|
|
// files[1] 300 .. 500
|
|
|
|
// Note that files are sorted by smallest key.
|
|
|
|
ASSERT_OK(Put("300", "v300"));
|
|
|
|
ASSERT_OK(Put("500", "v500"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_OK(Put("200", "v200"));
|
|
|
|
ASSERT_OK(Put("600", "v600"));
|
|
|
|
ASSERT_OK(Put("900", "v900"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("2,1,1", FilesPerLevel());
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Compact away the placeholder files we created initially
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
dbfull()->TEST_CompactRange(2, NULL, NULL);
|
|
|
|
ASSERT_EQ("2", FilesPerLevel());
|
2011-10-06 01:30:28 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Do a memtable compaction. Before bug-fix, the compaction would
|
|
|
|
// not detect the overlap with level-0 files and would incorrectly place
|
|
|
|
// the deletion in a deeper level.
|
|
|
|
ASSERT_OK(Delete("600"));
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
ASSERT_EQ("3", FilesPerLevel());
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get("600"));
|
|
|
|
} while (ChangeOptions());
|
2011-10-06 01:30:28 +02:00
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
TEST(DBTest, L0_CompactionBug_Issue44_a) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen();
|
|
|
|
ASSERT_OK(Put("b", "v"));
|
|
|
|
Reopen();
|
|
|
|
ASSERT_OK(Delete("b"));
|
|
|
|
ASSERT_OK(Delete("a"));
|
|
|
|
Reopen();
|
|
|
|
ASSERT_OK(Delete("a"));
|
|
|
|
Reopen();
|
|
|
|
ASSERT_OK(Put("a", "v"));
|
|
|
|
Reopen();
|
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("(a->v)", Contents());
|
|
|
|
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
|
|
|
ASSERT_EQ("(a->v)", Contents());
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, L0_CompactionBug_Issue44_b) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen();
|
|
|
|
Put("","");
|
|
|
|
Reopen();
|
|
|
|
Delete("e");
|
|
|
|
Put("","");
|
|
|
|
Reopen();
|
|
|
|
Put("c", "cv");
|
|
|
|
Reopen();
|
|
|
|
Put("","");
|
|
|
|
Reopen();
|
|
|
|
Put("","");
|
|
|
|
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
|
|
|
Reopen();
|
|
|
|
Put("d","dv");
|
|
|
|
Reopen();
|
|
|
|
Put("","");
|
|
|
|
Reopen();
|
|
|
|
Delete("d");
|
|
|
|
Delete("b");
|
|
|
|
Reopen();
|
|
|
|
ASSERT_EQ("(->)(c->cv)", Contents());
|
|
|
|
env_->SleepForMicroseconds(1000000); // Wait for compaction to finish
|
|
|
|
ASSERT_EQ("(->)(c->cv)", Contents());
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, ComparatorCheck) {
|
|
|
|
class NewComparator : public Comparator {
|
|
|
|
public:
|
|
|
|
virtual const char* Name() const { return "leveldb.NewComparator"; }
|
|
|
|
virtual int Compare(const Slice& a, const Slice& b) const {
|
|
|
|
return BytewiseComparator()->Compare(a, b);
|
|
|
|
}
|
|
|
|
virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
|
|
|
|
BytewiseComparator()->FindShortestSeparator(s, l);
|
|
|
|
}
|
|
|
|
virtual void FindShortSuccessor(std::string* key) const {
|
|
|
|
BytewiseComparator()->FindShortSuccessor(key);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
NewComparator cmp;
|
2012-04-17 17:36:46 +02:00
|
|
|
Options new_options = CurrentOptions();
|
2011-03-18 23:37:00 +01:00
|
|
|
new_options.comparator = &cmp;
|
|
|
|
Status s = TryReopen(&new_options);
|
|
|
|
ASSERT_TRUE(!s.ok());
|
|
|
|
ASSERT_TRUE(s.ToString().find("comparator") != std::string::npos)
|
|
|
|
<< s.ToString();
|
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
TEST(DBTest, CustomComparator) {
|
|
|
|
class NumberComparator : public Comparator {
|
|
|
|
public:
|
|
|
|
virtual const char* Name() const { return "test.NumberComparator"; }
|
|
|
|
virtual int Compare(const Slice& a, const Slice& b) const {
|
2011-11-14 18:06:16 +01:00
|
|
|
return ToNumber(a) - ToNumber(b);
|
|
|
|
}
|
|
|
|
virtual void FindShortestSeparator(std::string* s, const Slice& l) const {
|
|
|
|
ToNumber(*s); // Check format
|
|
|
|
ToNumber(l); // Check format
|
|
|
|
}
|
|
|
|
virtual void FindShortSuccessor(std::string* key) const {
|
|
|
|
ToNumber(*key); // Check format
|
|
|
|
}
|
|
|
|
private:
|
|
|
|
static int ToNumber(const Slice& x) {
|
|
|
|
// Check that there are no extra characters.
|
|
|
|
ASSERT_TRUE(x.size() >= 2 && x[0] == '[' && x[x.size()-1] == ']')
|
|
|
|
<< EscapeString(x);
|
|
|
|
int val;
|
|
|
|
char ignored;
|
|
|
|
ASSERT_TRUE(sscanf(x.ToString().c_str(), "[%i]%c", &val, &ignored) == 1)
|
|
|
|
<< EscapeString(x);
|
|
|
|
return val;
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
};
|
|
|
|
NumberComparator cmp;
|
2012-04-17 17:36:46 +02:00
|
|
|
Options new_options = CurrentOptions();
|
2011-10-31 18:22:06 +01:00
|
|
|
new_options.create_if_missing = true;
|
|
|
|
new_options.comparator = &cmp;
|
2012-04-17 17:36:46 +02:00
|
|
|
new_options.filter_policy = NULL; // Cannot use bloom filters
|
2011-11-14 18:06:16 +01:00
|
|
|
new_options.write_buffer_size = 1000; // Compact more often
|
2011-10-31 18:22:06 +01:00
|
|
|
DestroyAndReopen(&new_options);
|
2011-11-14 18:06:16 +01:00
|
|
|
ASSERT_OK(Put("[10]", "ten"));
|
|
|
|
ASSERT_OK(Put("[0x14]", "twenty"));
|
2011-10-31 18:22:06 +01:00
|
|
|
for (int i = 0; i < 2; i++) {
|
2011-11-14 18:06:16 +01:00
|
|
|
ASSERT_EQ("ten", Get("[10]"));
|
|
|
|
ASSERT_EQ("ten", Get("[0xa]"));
|
|
|
|
ASSERT_EQ("twenty", Get("[20]"));
|
|
|
|
ASSERT_EQ("twenty", Get("[0x14]"));
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_EQ("NOT_FOUND", Get("[15]"));
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get("[0xf]"));
|
2011-11-14 18:06:16 +01:00
|
|
|
Compact("[0]", "[9999]");
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int run = 0; run < 2; run++) {
|
|
|
|
for (int i = 0; i < 1000; i++) {
|
|
|
|
char buf[100];
|
|
|
|
snprintf(buf, sizeof(buf), "[%d]", i*10);
|
|
|
|
ASSERT_OK(Put(buf, buf));
|
|
|
|
}
|
|
|
|
Compact("[0]", "[1000000]");
|
2011-10-31 18:22:06 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-10-06 01:30:28 +02:00
|
|
|
TEST(DBTest, ManualCompaction) {
|
2012-06-23 04:30:03 +02:00
|
|
|
ASSERT_EQ(dbfull()->MaxMemCompactionLevel(), 2)
|
2011-10-06 01:30:28 +02:00
|
|
|
<< "Need to update this test to match kMaxMemCompactLevel";
|
|
|
|
|
|
|
|
MakeTables(3, "p", "q");
|
|
|
|
ASSERT_EQ("1,1,1", FilesPerLevel());
|
|
|
|
|
|
|
|
// Compaction range falls before files
|
|
|
|
Compact("", "c");
|
|
|
|
ASSERT_EQ("1,1,1", FilesPerLevel());
|
|
|
|
|
|
|
|
// Compaction range falls after files
|
|
|
|
Compact("r", "z");
|
|
|
|
ASSERT_EQ("1,1,1", FilesPerLevel());
|
|
|
|
|
|
|
|
// Compaction range overlaps files
|
|
|
|
Compact("p1", "p9");
|
|
|
|
ASSERT_EQ("0,0,1", FilesPerLevel());
|
|
|
|
|
|
|
|
// Populate a different range
|
|
|
|
MakeTables(3, "c", "e");
|
|
|
|
ASSERT_EQ("1,1,2", FilesPerLevel());
|
|
|
|
|
|
|
|
// Compact just the new range
|
|
|
|
Compact("b", "f");
|
|
|
|
ASSERT_EQ("0,0,2", FilesPerLevel());
|
|
|
|
|
|
|
|
// Compact all
|
|
|
|
MakeTables(1, "a", "z");
|
|
|
|
ASSERT_EQ("0,1,2", FilesPerLevel());
|
|
|
|
db_->CompactRange(NULL, NULL);
|
|
|
|
ASSERT_EQ("0,0,1", FilesPerLevel());
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
TEST(DBTest, DBOpen_Options) {
|
|
|
|
std::string dbname = test::TmpDir() + "/db_options_test";
|
|
|
|
DestroyDB(dbname, Options());
|
|
|
|
|
|
|
|
// Does not exist, and create_if_missing == false: error
|
|
|
|
DB* db = NULL;
|
|
|
|
Options opts;
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
Status s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != NULL);
|
|
|
|
ASSERT_TRUE(db == NULL);
|
|
|
|
|
|
|
|
// Does not exist, and create_if_missing == true: OK
|
|
|
|
opts.create_if_missing = true;
|
|
|
|
s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_OK(s);
|
|
|
|
ASSERT_TRUE(db != NULL);
|
|
|
|
|
|
|
|
delete db;
|
|
|
|
db = NULL;
|
|
|
|
|
|
|
|
// Does exist, and error_if_exists == true: error
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
opts.error_if_exists = true;
|
|
|
|
s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != NULL);
|
|
|
|
ASSERT_TRUE(db == NULL);
|
|
|
|
|
|
|
|
// Does exist, and error_if_exists == false: OK
|
|
|
|
opts.create_if_missing = true;
|
|
|
|
opts.error_if_exists = false;
|
|
|
|
s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_OK(s);
|
|
|
|
ASSERT_TRUE(db != NULL);
|
|
|
|
|
|
|
|
delete db;
|
|
|
|
db = NULL;
|
|
|
|
}
|
|
|
|
|
2012-10-29 23:25:01 +01:00
|
|
|
TEST(DBTest, DBOpen_Change_NumLevels) {
|
|
|
|
std::string dbname = test::TmpDir() + "/db_change_num_levels";
|
|
|
|
DestroyDB(dbname, Options());
|
|
|
|
Options opts;
|
|
|
|
Status s;
|
2012-10-30 00:26:20 +01:00
|
|
|
DB* db = NULL;
|
2012-10-29 23:25:01 +01:00
|
|
|
opts.create_if_missing = true;
|
2012-10-30 00:26:20 +01:00
|
|
|
s = DB::Open(opts, dbname, &db);
|
2012-10-29 23:25:01 +01:00
|
|
|
ASSERT_OK(s);
|
2012-10-30 00:26:20 +01:00
|
|
|
ASSERT_TRUE(db != NULL);
|
|
|
|
db->Put(WriteOptions(), "a", "123");
|
|
|
|
db->Put(WriteOptions(), "b", "234");
|
|
|
|
db->CompactRange(NULL, NULL);
|
|
|
|
delete db;
|
|
|
|
db = NULL;
|
2012-10-29 23:25:01 +01:00
|
|
|
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
opts.num_levels = 2;
|
2012-10-30 00:26:20 +01:00
|
|
|
s = DB::Open(opts, dbname, &db);
|
2012-10-29 23:25:01 +01:00
|
|
|
ASSERT_TRUE(strstr(s.ToString().c_str(), "Corruption") != NULL);
|
2012-10-30 00:26:20 +01:00
|
|
|
ASSERT_TRUE(db == NULL);
|
2012-10-29 23:25:01 +01:00
|
|
|
}
|
|
|
|
|
2012-01-25 23:56:52 +01:00
|
|
|
// Check that number of files does not grow when we are out of space
|
|
|
|
TEST(DBTest, NoSpace) {
|
2012-04-17 17:36:46 +02:00
|
|
|
Options options = CurrentOptions();
|
2012-01-25 23:56:52 +01:00
|
|
|
options.env = env_;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
ASSERT_EQ("v1", Get("foo"));
|
|
|
|
Compact("a", "z");
|
|
|
|
const int num_files = CountFiles();
|
|
|
|
env_->no_space_.Release_Store(env_); // Force out-of-space errors
|
2012-08-27 08:45:35 +02:00
|
|
|
env_->sleep_counter_.Reset();
|
2012-08-23 01:57:51 +02:00
|
|
|
for (int i = 0; i < 5; i++) {
|
2012-06-23 04:30:03 +02:00
|
|
|
for (int level = 0; level < dbfull()->NumberLevels()-1; level++) {
|
2012-01-25 23:56:52 +01:00
|
|
|
dbfull()->TEST_CompactRange(level, NULL, NULL);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
env_->no_space_.Release_Store(NULL);
|
2012-08-23 01:57:51 +02:00
|
|
|
ASSERT_LT(CountFiles(), num_files + 3);
|
|
|
|
|
|
|
|
// Check that compaction attempts slept after errors
|
|
|
|
ASSERT_GE(env_->sleep_counter_.Read(), 5);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, NonWritableFileSystem)
|
|
|
|
{
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 1000;
|
|
|
|
options.env = env_;
|
|
|
|
Reopen(&options);
|
|
|
|
ASSERT_OK(Put("foo", "v1"));
|
|
|
|
env_->non_writable_.Release_Store(env_); // Force errors for new files
|
|
|
|
std::string big(100000, 'x');
|
|
|
|
int errors = 0;
|
|
|
|
for (int i = 0; i < 20; i++) {
|
|
|
|
fprintf(stderr, "iter %d; errors %d\n", i, errors);
|
|
|
|
if (!Put("foo", big).ok()) {
|
|
|
|
errors++;
|
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ASSERT_GT(errors, 0);
|
|
|
|
env_->non_writable_.Release_Store(NULL);
|
2012-01-25 23:56:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, FilesDeletedAfterCompaction) {
|
|
|
|
ASSERT_OK(Put("foo", "v2"));
|
|
|
|
Compact("a", "z");
|
2012-10-29 19:12:24 +01:00
|
|
|
const int num_files = CountLiveFiles();
|
2012-01-25 23:56:52 +01:00
|
|
|
for (int i = 0; i < 10; i++) {
|
|
|
|
ASSERT_OK(Put("foo", "v2"));
|
|
|
|
Compact("a", "z");
|
|
|
|
}
|
2012-10-29 19:12:24 +01:00
|
|
|
ASSERT_EQ(CountLiveFiles(), num_files);
|
2012-01-25 23:56:52 +01:00
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
TEST(DBTest, BloomFilter) {
|
|
|
|
env_->count_random_reads_ = true;
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.env = env_;
|
|
|
|
options.block_cache = NewLRUCache(0); // Prevent cache hits
|
|
|
|
options.filter_policy = NewBloomFilterPolicy(10);
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
// Populate multiple layers
|
|
|
|
const int N = 10000;
|
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_OK(Put(Key(i), Key(i)));
|
|
|
|
}
|
|
|
|
Compact("a", "z");
|
|
|
|
for (int i = 0; i < N; i += 100) {
|
|
|
|
ASSERT_OK(Put(Key(i), Key(i)));
|
|
|
|
}
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
|
|
|
|
// Prevent auto compactions triggered by seeks
|
|
|
|
env_->delay_sstable_sync_.Release_Store(env_);
|
|
|
|
|
|
|
|
// Lookup present keys. Should rarely read from small sstable.
|
|
|
|
env_->random_read_counter_.Reset();
|
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_EQ(Key(i), Get(Key(i)));
|
|
|
|
}
|
|
|
|
int reads = env_->random_read_counter_.Read();
|
|
|
|
fprintf(stderr, "%d present => %d reads\n", N, reads);
|
|
|
|
ASSERT_GE(reads, N);
|
|
|
|
ASSERT_LE(reads, N + 2*N/100);
|
|
|
|
|
|
|
|
// Lookup present keys. Should rarely read from either sstable.
|
|
|
|
env_->random_read_counter_.Reset();
|
|
|
|
for (int i = 0; i < N; i++) {
|
|
|
|
ASSERT_EQ("NOT_FOUND", Get(Key(i) + ".missing"));
|
|
|
|
}
|
|
|
|
reads = env_->random_read_counter_.Read();
|
|
|
|
fprintf(stderr, "%d missing => %d reads\n", N, reads);
|
|
|
|
ASSERT_LE(reads, 3*N/100);
|
|
|
|
|
|
|
|
env_->delay_sstable_sync_.Release_Store(NULL);
|
|
|
|
Close();
|
|
|
|
delete options.block_cache;
|
|
|
|
delete options.filter_policy;
|
|
|
|
}
|
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
TEST(DBTest, SnapshotFiles) {
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.write_buffer_size = 100000000; // Large write buffer
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Write 8MB (80 values, each 100K)
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 80; i++) {
|
|
|
|
values.push_back(RandomString(&rnd, 100000));
|
|
|
|
ASSERT_OK(Put(Key(i), values[i]));
|
|
|
|
}
|
|
|
|
|
|
|
|
// assert that nothing makes it to disk yet.
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
|
|
|
|
// get a file snapshot
|
2012-09-24 23:01:01 +02:00
|
|
|
uint64_t manifest_number = 0;
|
|
|
|
uint64_t manifest_size = 0;
|
2012-09-15 02:11:35 +02:00
|
|
|
std::vector<std::string> files;
|
|
|
|
dbfull()->DisableFileDeletions();
|
2012-09-24 23:01:01 +02:00
|
|
|
dbfull()->GetLiveFiles(files, &manifest_size);
|
2012-09-15 02:11:35 +02:00
|
|
|
|
|
|
|
// CURRENT, MANIFEST, *.sst files
|
2012-11-06 21:02:18 +01:00
|
|
|
ASSERT_EQ(files.size(), 3U);
|
2012-09-15 02:11:35 +02:00
|
|
|
|
2012-09-24 23:01:01 +02:00
|
|
|
uint64_t number = 0;
|
|
|
|
FileType type;
|
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
// copy these files to a new snapshot directory
|
|
|
|
std::string snapdir = dbname_ + ".snapdir/";
|
|
|
|
std::string mkdir = "mkdir -p " + snapdir;
|
|
|
|
ASSERT_EQ(system(mkdir.c_str()), 0);
|
2012-09-24 23:01:01 +02:00
|
|
|
|
|
|
|
for (unsigned int i = 0; i < files.size(); i++) {
|
2012-09-15 02:11:35 +02:00
|
|
|
std::string src = dbname_ + "/" + files[i];
|
|
|
|
std::string dest = snapdir + "/" + files[i];
|
2012-09-24 23:01:01 +02:00
|
|
|
|
|
|
|
uint64_t size;
|
|
|
|
ASSERT_OK(env_->GetFileSize(src, &size));
|
|
|
|
|
2012-11-01 18:50:08 +01:00
|
|
|
// record the number and the size of the
|
2012-09-24 23:01:01 +02:00
|
|
|
// latest manifest file
|
|
|
|
if (ParseFileName(files[i].substr(1), &number, &type)) {
|
|
|
|
if (type == kDescriptorFile) {
|
|
|
|
if (number > manifest_number) {
|
|
|
|
manifest_number = number;
|
|
|
|
ASSERT_GE(size, manifest_size);
|
|
|
|
size = manifest_size; // copy only valid MANIFEST data
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
SequentialFile* srcfile;
|
|
|
|
ASSERT_OK(env_->NewSequentialFile(src, &srcfile));
|
|
|
|
WritableFile* destfile;
|
|
|
|
ASSERT_OK(env_->NewWritableFile(dest, &destfile));
|
2012-11-01 18:50:08 +01:00
|
|
|
|
2012-09-24 23:01:01 +02:00
|
|
|
char buffer[4096];
|
|
|
|
Slice slice;
|
|
|
|
while (size > 0) {
|
|
|
|
uint64_t one = std::min(sizeof(buffer), size);
|
|
|
|
ASSERT_OK(srcfile->Read(one, &slice, buffer));
|
|
|
|
ASSERT_OK(destfile->Append(slice));
|
|
|
|
size -= slice.size();
|
|
|
|
}
|
|
|
|
ASSERT_OK(destfile->Close());
|
|
|
|
delete destfile;
|
|
|
|
delete srcfile;
|
2012-09-15 02:11:35 +02:00
|
|
|
}
|
2012-09-24 23:01:01 +02:00
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
// release file snapshot
|
|
|
|
dbfull()->DisableFileDeletions();
|
|
|
|
|
2012-09-24 23:01:01 +02:00
|
|
|
// overwrite one key, this key should not appear in the snapshot
|
|
|
|
std::vector<std::string> extras;
|
|
|
|
for (unsigned int i = 0; i < 1; i++) {
|
|
|
|
extras.push_back(RandomString(&rnd, 100000));
|
|
|
|
ASSERT_OK(Put(Key(i), extras[i]));
|
|
|
|
}
|
2012-11-01 18:50:08 +01:00
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
// verify that data in the snapshot are correct
|
|
|
|
Options opts;
|
|
|
|
DB* snapdb;
|
|
|
|
opts.create_if_missing = false;
|
|
|
|
Status stat = DB::Open(opts, snapdir, &snapdb);
|
|
|
|
ASSERT_TRUE(stat.ok());
|
|
|
|
|
|
|
|
ReadOptions roptions;
|
|
|
|
std::string val;
|
2012-09-24 23:01:01 +02:00
|
|
|
for (unsigned int i = 0; i < 80; i++) {
|
2012-09-15 02:11:35 +02:00
|
|
|
stat = snapdb->Get(roptions, Key(i), &val);
|
|
|
|
ASSERT_EQ(values[i].compare(val), 0);
|
|
|
|
}
|
|
|
|
delete snapdb;
|
2012-09-24 23:01:01 +02:00
|
|
|
|
2012-11-01 18:50:08 +01:00
|
|
|
// look at the new live files after we added an 'extra' key
|
2012-09-24 23:01:01 +02:00
|
|
|
// and after we took the first snapshot.
|
|
|
|
uint64_t new_manifest_number = 0;
|
|
|
|
uint64_t new_manifest_size = 0;
|
|
|
|
std::vector<std::string> newfiles;
|
|
|
|
dbfull()->DisableFileDeletions();
|
|
|
|
dbfull()->GetLiveFiles(newfiles, &new_manifest_size);
|
|
|
|
|
|
|
|
// find the new manifest file. assert that this manifest file is
|
|
|
|
// the same one as in the previous snapshot. But its size should be
|
|
|
|
// larger because we added an extra key after taking the
|
|
|
|
// previous shapshot.
|
|
|
|
for (unsigned int i = 0; i < newfiles.size(); i++) {
|
|
|
|
std::string src = dbname_ + "/" + newfiles[i];
|
2012-11-01 18:50:08 +01:00
|
|
|
// record the lognumber and the size of the
|
2012-09-24 23:01:01 +02:00
|
|
|
// latest manifest file
|
|
|
|
if (ParseFileName(newfiles[i].substr(1), &number, &type)) {
|
|
|
|
if (type == kDescriptorFile) {
|
|
|
|
if (number > new_manifest_number) {
|
|
|
|
uint64_t size;
|
|
|
|
new_manifest_number = number;
|
|
|
|
ASSERT_OK(env_->GetFileSize(src, &size));
|
|
|
|
ASSERT_GE(size, new_manifest_size);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ASSERT_EQ(manifest_number, new_manifest_number);
|
|
|
|
ASSERT_GT(new_manifest_size, manifest_size);
|
2012-11-01 18:50:08 +01:00
|
|
|
|
2012-09-24 23:01:01 +02:00
|
|
|
// release file snapshot
|
|
|
|
dbfull()->DisableFileDeletions();
|
2012-09-15 02:11:35 +02:00
|
|
|
}
|
|
|
|
|
2012-11-26 22:56:45 +01:00
|
|
|
void ListLogFiles(Env* env,
|
|
|
|
const std::string& path,
|
|
|
|
std::vector<uint64_t>* logFiles) {
|
|
|
|
std::vector<std::string> files;
|
|
|
|
env->GetChildren(path, &files);
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
for (size_t i = 0; i < files.size(); ++i) {
|
|
|
|
if (ParseFileName(files[i], &number, &type)) {
|
|
|
|
if (type == kLogFile) {
|
|
|
|
logFiles->push_back(number);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, WALArchival) {
|
|
|
|
std::string value(1024, '1');
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.WAL_ttl_seconds = 1000;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
|
|
|
|
|
|
|
|
// TEST : Create DB with a ttl.
|
|
|
|
// Put some keys. Count the log files present in the DB just after insert.
|
|
|
|
// Re-open db. Causes deletion/archival to take place.
|
|
|
|
// Assert that the files moved under "/archive".
|
|
|
|
|
2012-12-08 01:30:22 +01:00
|
|
|
std::string archiveDir = ArchivalDirectory(dbname_);
|
2012-11-26 22:56:45 +01:00
|
|
|
|
|
|
|
for (int i = 0; i < 10; ++i) {
|
|
|
|
|
|
|
|
for (int j = 0; j < 10; ++j) {
|
|
|
|
ASSERT_OK(Put(Key(10*i+j), value));
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<uint64_t> logFiles;
|
|
|
|
ListLogFiles(env_, dbname_, &logFiles);
|
|
|
|
|
|
|
|
options.create_if_missing = false;
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
std::vector<uint64_t> logs;
|
|
|
|
ListLogFiles(env_, archiveDir, &logs);
|
|
|
|
std::set<uint64_t> archivedFiles(logs.begin(), logs.end());
|
|
|
|
|
|
|
|
for (std::vector<uint64_t>::iterator it = logFiles.begin();
|
|
|
|
it != logFiles.end();
|
|
|
|
++it) {
|
|
|
|
ASSERT_TRUE(archivedFiles.find(*it) != archivedFiles.end());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// REOPEN database with 0 TTL. all files should have been deleted.
|
|
|
|
std::vector<uint64_t> logFiles;
|
|
|
|
ListLogFiles(env_, archiveDir, &logFiles);
|
|
|
|
ASSERT_TRUE(logFiles.size() > 0);
|
|
|
|
options.WAL_ttl_seconds = 1;
|
|
|
|
env_->SleepForMicroseconds(2*1000*1000);
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
logFiles.clear();
|
|
|
|
ListLogFiles(env_, archiveDir, &logFiles);
|
|
|
|
ASSERT_TRUE(logFiles.size() == 0);
|
|
|
|
|
|
|
|
}
|
2012-09-27 10:05:38 +02:00
|
|
|
|
2012-11-30 02:28:37 +01:00
|
|
|
TEST(DBTest, TransactionLogIterator) {
|
|
|
|
std::string value(1024, '1');
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.WAL_ttl_seconds = 1000;
|
|
|
|
DestroyAndReopen(&options);
|
|
|
|
Put("key1", value);
|
|
|
|
Put("key2", value);
|
|
|
|
Put("key2", value);
|
2012-12-11 00:37:00 +01:00
|
|
|
ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 3U);
|
2012-11-30 02:28:37 +01:00
|
|
|
{
|
|
|
|
TransactionLogIterator* iter;
|
|
|
|
Status status = dbfull()->GetUpdatesSince(0, &iter);
|
|
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
iter->Next();
|
|
|
|
int i = 0;
|
|
|
|
SequenceNumber lastSequence = 0;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
WriteBatch batch;
|
|
|
|
iter->GetBatch(&batch);
|
|
|
|
SequenceNumber current = WriteBatchInternal::Sequence(&batch);
|
2012-12-10 20:02:07 +01:00
|
|
|
ASSERT_TRUE(current > lastSequence);
|
2012-11-30 02:28:37 +01:00
|
|
|
++i;
|
|
|
|
lastSequence = current;
|
|
|
|
ASSERT_TRUE(iter->status().ok());
|
|
|
|
iter->Next();
|
|
|
|
}
|
|
|
|
ASSERT_EQ(i, 3);
|
|
|
|
}
|
|
|
|
Reopen(&options);
|
|
|
|
{
|
|
|
|
Put("key4", value);
|
|
|
|
Put("key5", value);
|
|
|
|
Put("key6", value);
|
|
|
|
}
|
|
|
|
{
|
|
|
|
TransactionLogIterator* iter;
|
|
|
|
Status status = dbfull()->GetUpdatesSince(0, &iter);
|
|
|
|
ASSERT_TRUE(status.ok());
|
|
|
|
ASSERT_TRUE(!iter->Valid());
|
|
|
|
iter->Next();
|
|
|
|
int i = 0;
|
|
|
|
SequenceNumber lastSequence = 0;
|
|
|
|
while (iter->Valid()) {
|
|
|
|
WriteBatch batch;
|
|
|
|
iter->GetBatch(&batch);
|
|
|
|
SequenceNumber current = WriteBatchInternal::Sequence(&batch);
|
|
|
|
ASSERT_TRUE(current > lastSequence);
|
|
|
|
lastSequence = current;
|
|
|
|
ASSERT_TRUE(iter->status().ok());
|
|
|
|
iter->Next();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
ASSERT_EQ(i, 6);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-09-27 10:05:38 +02:00
|
|
|
TEST(DBTest, ReadCompaction) {
|
|
|
|
std::string value(4096, '4'); // a string of size 4K
|
|
|
|
{
|
|
|
|
Options options = CurrentOptions();
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.max_open_files = 20; // only 10 file in file-cache
|
|
|
|
options.target_file_size_base = 512;
|
|
|
|
options.write_buffer_size = 64 * 1024;
|
|
|
|
options.filter_policy = NULL;
|
|
|
|
options.block_size = 4096;
|
|
|
|
options.block_cache = NewLRUCache(0); // Prevent cache hits
|
|
|
|
|
|
|
|
Reopen(&options);
|
|
|
|
|
|
|
|
// Write 8MB (2000 values, each 4K)
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
std::vector<std::string> values;
|
|
|
|
for (int i = 0; i < 2000; i++) {
|
|
|
|
ASSERT_OK(Put(Key(i), value));
|
|
|
|
}
|
|
|
|
|
|
|
|
// clear level 0 and 1 if necessary.
|
|
|
|
dbfull()->TEST_CompactMemTable();
|
|
|
|
dbfull()->TEST_CompactRange(0, NULL, NULL);
|
|
|
|
dbfull()->TEST_CompactRange(1, NULL, NULL);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_EQ(NumTableFilesAtLevel(1), 0);
|
|
|
|
|
|
|
|
// write some new keys into level 0
|
|
|
|
for (int i = 0; i < 2000; i = i + 16) {
|
|
|
|
ASSERT_OK(Put(Key(i), value));
|
|
|
|
}
|
|
|
|
dbfull()->Flush(FlushOptions());
|
|
|
|
|
|
|
|
// Wait for any write compaction to finish
|
|
|
|
dbfull()->TEST_WaitForCompact();
|
|
|
|
|
|
|
|
// remember number of files in each level
|
|
|
|
int l1 = NumTableFilesAtLevel(0);
|
|
|
|
int l2 = NumTableFilesAtLevel(1);
|
|
|
|
int l3 = NumTableFilesAtLevel(3);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(0), 0);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(1), 0);
|
|
|
|
ASSERT_NE(NumTableFilesAtLevel(2), 0);
|
|
|
|
|
|
|
|
// read a bunch of times, trigger read compaction
|
|
|
|
for (int j = 0; j < 100; j++) {
|
|
|
|
for (int i = 0; i < 2000; i++) {
|
|
|
|
Get(Key(i));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// wait for read compaction to finish
|
|
|
|
env_->SleepForMicroseconds(1000000);
|
|
|
|
|
|
|
|
// verify that the number of files have decreased
|
|
|
|
// in some level, indicating that there was a compaction
|
|
|
|
ASSERT_TRUE(NumTableFilesAtLevel(0) < l1 ||
|
|
|
|
NumTableFilesAtLevel(1) < l2 ||
|
2012-11-01 18:50:08 +01:00
|
|
|
NumTableFilesAtLevel(2) < l3);
|
2012-09-27 10:05:38 +02:00
|
|
|
delete options.block_cache;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-05-28 02:53:58 +02:00
|
|
|
// Multi-threaded test:
|
|
|
|
namespace {
|
|
|
|
|
|
|
|
static const int kNumThreads = 4;
|
|
|
|
static const int kTestSeconds = 10;
|
|
|
|
static const int kNumKeys = 1000;
|
|
|
|
|
|
|
|
struct MTState {
|
|
|
|
DBTest* test;
|
|
|
|
port::AtomicPointer stop;
|
|
|
|
port::AtomicPointer counter[kNumThreads];
|
|
|
|
port::AtomicPointer thread_done[kNumThreads];
|
|
|
|
};
|
|
|
|
|
|
|
|
struct MTThread {
|
|
|
|
MTState* state;
|
|
|
|
int id;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void MTThreadBody(void* arg) {
|
|
|
|
MTThread* t = reinterpret_cast<MTThread*>(arg);
|
2012-01-25 23:56:52 +01:00
|
|
|
int id = t->id;
|
2011-05-28 02:53:58 +02:00
|
|
|
DB* db = t->state->test->db_;
|
|
|
|
uintptr_t counter = 0;
|
2012-01-25 23:56:52 +01:00
|
|
|
fprintf(stderr, "... starting thread %d\n", id);
|
|
|
|
Random rnd(1000 + id);
|
2011-05-28 02:53:58 +02:00
|
|
|
std::string value;
|
|
|
|
char valbuf[1500];
|
|
|
|
while (t->state->stop.Acquire_Load() == NULL) {
|
2012-01-25 23:56:52 +01:00
|
|
|
t->state->counter[id].Release_Store(reinterpret_cast<void*>(counter));
|
2011-05-28 02:53:58 +02:00
|
|
|
|
|
|
|
int key = rnd.Uniform(kNumKeys);
|
|
|
|
char keybuf[20];
|
|
|
|
snprintf(keybuf, sizeof(keybuf), "%016d", key);
|
|
|
|
|
|
|
|
if (rnd.OneIn(2)) {
|
|
|
|
// Write values of the form <key, my id, counter>.
|
|
|
|
// We add some padding for force compactions.
|
|
|
|
snprintf(valbuf, sizeof(valbuf), "%d.%d.%-1000d",
|
2012-01-25 23:56:52 +01:00
|
|
|
key, id, static_cast<int>(counter));
|
2011-05-28 02:53:58 +02:00
|
|
|
ASSERT_OK(db->Put(WriteOptions(), Slice(keybuf), Slice(valbuf)));
|
|
|
|
} else {
|
|
|
|
// Read a value and verify that it matches the pattern written above.
|
|
|
|
Status s = db->Get(ReadOptions(), Slice(keybuf), &value);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
// Key has not yet been written
|
|
|
|
} else {
|
|
|
|
// Check that the writer thread counter is >= the counter in the value
|
|
|
|
ASSERT_OK(s);
|
|
|
|
int k, w, c;
|
|
|
|
ASSERT_EQ(3, sscanf(value.c_str(), "%d.%d.%d", &k, &w, &c)) << value;
|
|
|
|
ASSERT_EQ(k, key);
|
|
|
|
ASSERT_GE(w, 0);
|
|
|
|
ASSERT_LT(w, kNumThreads);
|
2012-11-06 21:02:18 +01:00
|
|
|
ASSERT_LE((unsigned int)c, reinterpret_cast<uintptr_t>(
|
2011-05-28 02:53:58 +02:00
|
|
|
t->state->counter[w].Acquire_Load()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
counter++;
|
|
|
|
}
|
2012-01-25 23:56:52 +01:00
|
|
|
t->state->thread_done[id].Release_Store(t);
|
|
|
|
fprintf(stderr, "... stopping thread %d after %d ops\n", id, int(counter));
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
} // namespace
|
2011-05-28 02:53:58 +02:00
|
|
|
|
|
|
|
TEST(DBTest, MultiThreaded) {
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
// Initialize state
|
|
|
|
MTState mt;
|
|
|
|
mt.test = this;
|
|
|
|
mt.stop.Release_Store(0);
|
|
|
|
for (int id = 0; id < kNumThreads; id++) {
|
|
|
|
mt.counter[id].Release_Store(0);
|
|
|
|
mt.thread_done[id].Release_Store(0);
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Start threads
|
|
|
|
MTThread thread[kNumThreads];
|
|
|
|
for (int id = 0; id < kNumThreads; id++) {
|
|
|
|
thread[id].state = &mt;
|
|
|
|
thread[id].id = id;
|
|
|
|
env_->StartThread(MTThreadBody, &thread[id]);
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Let them run for a while
|
|
|
|
env_->SleepForMicroseconds(kTestSeconds * 1000000);
|
2011-05-28 02:53:58 +02:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
// Stop the threads and wait for them to finish
|
|
|
|
mt.stop.Release_Store(&mt);
|
|
|
|
for (int id = 0; id < kNumThreads; id++) {
|
|
|
|
while (mt.thread_done[id].Acquire_Load() == NULL) {
|
|
|
|
env_->SleepForMicroseconds(100000);
|
|
|
|
}
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
} while (ChangeOptions());
|
2011-05-28 02:53:58 +02:00
|
|
|
}
|
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
namespace {
|
|
|
|
typedef std::map<std::string, std::string> KVMap;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
class ModelDB: public DB {
|
|
|
|
public:
|
2011-05-21 04:17:43 +02:00
|
|
|
class ModelSnapshot : public Snapshot {
|
|
|
|
public:
|
|
|
|
KVMap map_;
|
|
|
|
};
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
explicit ModelDB(const Options& options): options_(options) { }
|
|
|
|
~ModelDB() { }
|
|
|
|
virtual Status Put(const WriteOptions& o, const Slice& k, const Slice& v) {
|
|
|
|
return DB::Put(o, k, v);
|
|
|
|
}
|
|
|
|
virtual Status Delete(const WriteOptions& o, const Slice& key) {
|
|
|
|
return DB::Delete(o, key);
|
|
|
|
}
|
|
|
|
virtual Status Get(const ReadOptions& options,
|
|
|
|
const Slice& key, std::string* value) {
|
|
|
|
assert(false); // Not implemented
|
|
|
|
return Status::NotFound(key);
|
|
|
|
}
|
|
|
|
virtual Iterator* NewIterator(const ReadOptions& options) {
|
|
|
|
if (options.snapshot == NULL) {
|
|
|
|
KVMap* saved = new KVMap;
|
|
|
|
*saved = map_;
|
|
|
|
return new ModelIter(saved, true);
|
|
|
|
} else {
|
|
|
|
const KVMap* snapshot_state =
|
2011-05-21 04:17:43 +02:00
|
|
|
&(reinterpret_cast<const ModelSnapshot*>(options.snapshot)->map_);
|
2011-03-18 23:37:00 +01:00
|
|
|
return new ModelIter(snapshot_state, false);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
virtual const Snapshot* GetSnapshot() {
|
2011-05-21 04:17:43 +02:00
|
|
|
ModelSnapshot* snapshot = new ModelSnapshot;
|
|
|
|
snapshot->map_ = map_;
|
|
|
|
return snapshot;
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual void ReleaseSnapshot(const Snapshot* snapshot) {
|
2011-05-21 04:17:43 +02:00
|
|
|
delete reinterpret_cast<const ModelSnapshot*>(snapshot);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
virtual Status Write(const WriteOptions& options, WriteBatch* batch) {
|
2011-05-21 04:17:43 +02:00
|
|
|
class Handler : public WriteBatch::Handler {
|
|
|
|
public:
|
|
|
|
KVMap* map_;
|
|
|
|
virtual void Put(const Slice& key, const Slice& value) {
|
|
|
|
(*map_)[key.ToString()] = value.ToString();
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2011-05-21 04:17:43 +02:00
|
|
|
virtual void Delete(const Slice& key) {
|
|
|
|
map_->erase(key.ToString());
|
|
|
|
}
|
|
|
|
};
|
|
|
|
Handler handler;
|
|
|
|
handler.map_ = &map_;
|
|
|
|
return batch->Iterate(&handler);
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-04-12 21:38:58 +02:00
|
|
|
virtual bool GetProperty(const Slice& property, std::string* value) {
|
2011-03-18 23:37:00 +01:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes) {
|
|
|
|
for (int i = 0; i < n; i++) {
|
|
|
|
sizes[i] = 0;
|
|
|
|
}
|
|
|
|
}
|
2011-10-06 01:30:28 +02:00
|
|
|
virtual void CompactRange(const Slice* start, const Slice* end) {
|
|
|
|
}
|
|
|
|
|
2012-06-23 04:30:03 +02:00
|
|
|
virtual int NumberLevels()
|
|
|
|
{
|
2012-11-29 01:42:36 +01:00
|
|
|
return 1;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual int MaxMemCompactionLevel()
|
|
|
|
{
|
2012-11-29 01:42:36 +01:00
|
|
|
return 1;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
virtual int Level0StopWriteTrigger()
|
|
|
|
{
|
2012-11-29 01:42:36 +01:00
|
|
|
return -1;
|
2012-06-23 04:30:03 +02:00
|
|
|
}
|
|
|
|
|
2012-07-06 20:42:09 +02:00
|
|
|
virtual Status Flush(const leveldb::FlushOptions& options) {
|
|
|
|
Status ret;
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-09-15 02:11:35 +02:00
|
|
|
virtual Status DisableFileDeletions() {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
virtual Status EnableFileDeletions() {
|
|
|
|
return Status::OK();
|
|
|
|
}
|
2012-09-24 23:01:01 +02:00
|
|
|
virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size) {
|
2012-09-15 02:11:35 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2012-12-11 00:37:00 +01:00
|
|
|
virtual SequenceNumber GetLatestSequenceNumber() {
|
|
|
|
return 0;
|
|
|
|
}
|
2012-11-30 02:28:37 +01:00
|
|
|
virtual Status GetUpdatesSince(leveldb::SequenceNumber,
|
|
|
|
leveldb::TransactionLogIterator**) {
|
|
|
|
return Status::NotSupported("Not supported in Model DB");
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
private:
|
|
|
|
class ModelIter: public Iterator {
|
|
|
|
public:
|
|
|
|
ModelIter(const KVMap* map, bool owned)
|
|
|
|
: map_(map), owned_(owned), iter_(map_->end()) {
|
|
|
|
}
|
|
|
|
~ModelIter() {
|
|
|
|
if (owned_) delete map_;
|
|
|
|
}
|
|
|
|
virtual bool Valid() const { return iter_ != map_->end(); }
|
|
|
|
virtual void SeekToFirst() { iter_ = map_->begin(); }
|
|
|
|
virtual void SeekToLast() {
|
|
|
|
if (map_->empty()) {
|
|
|
|
iter_ = map_->end();
|
|
|
|
} else {
|
|
|
|
iter_ = map_->find(map_->rbegin()->first);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
virtual void Seek(const Slice& k) {
|
|
|
|
iter_ = map_->lower_bound(k.ToString());
|
|
|
|
}
|
|
|
|
virtual void Next() { ++iter_; }
|
|
|
|
virtual void Prev() { --iter_; }
|
|
|
|
virtual Slice key() const { return iter_->first; }
|
|
|
|
virtual Slice value() const { return iter_->second; }
|
|
|
|
virtual Status status() const { return Status::OK(); }
|
|
|
|
private:
|
|
|
|
const KVMap* const map_;
|
|
|
|
const bool owned_; // Do we own map_
|
|
|
|
KVMap::const_iterator iter_;
|
|
|
|
};
|
|
|
|
const Options options_;
|
|
|
|
KVMap map_;
|
|
|
|
};
|
|
|
|
|
|
|
|
static std::string RandomKey(Random* rnd) {
|
|
|
|
int len = (rnd->OneIn(3)
|
|
|
|
? 1 // Short sometimes to encourage collisions
|
|
|
|
: (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
|
|
|
|
return test::RandomKey(rnd, len);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool CompareIterators(int step,
|
|
|
|
DB* model,
|
|
|
|
DB* db,
|
|
|
|
const Snapshot* model_snap,
|
|
|
|
const Snapshot* db_snap) {
|
|
|
|
ReadOptions options;
|
|
|
|
options.snapshot = model_snap;
|
|
|
|
Iterator* miter = model->NewIterator(options);
|
|
|
|
options.snapshot = db_snap;
|
|
|
|
Iterator* dbiter = db->NewIterator(options);
|
|
|
|
bool ok = true;
|
|
|
|
int count = 0;
|
|
|
|
for (miter->SeekToFirst(), dbiter->SeekToFirst();
|
|
|
|
ok && miter->Valid() && dbiter->Valid();
|
|
|
|
miter->Next(), dbiter->Next()) {
|
|
|
|
count++;
|
|
|
|
if (miter->key().compare(dbiter->key()) != 0) {
|
|
|
|
fprintf(stderr, "step %d: Key mismatch: '%s' vs. '%s'\n",
|
|
|
|
step,
|
|
|
|
EscapeString(miter->key()).c_str(),
|
|
|
|
EscapeString(dbiter->key()).c_str());
|
|
|
|
ok = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (miter->value().compare(dbiter->value()) != 0) {
|
|
|
|
fprintf(stderr, "step %d: Value mismatch for key '%s': '%s' vs. '%s'\n",
|
|
|
|
step,
|
|
|
|
EscapeString(miter->key()).c_str(),
|
|
|
|
EscapeString(miter->value()).c_str(),
|
|
|
|
EscapeString(miter->value()).c_str());
|
|
|
|
ok = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ok) {
|
|
|
|
if (miter->Valid() != dbiter->Valid()) {
|
|
|
|
fprintf(stderr, "step %d: Mismatch at end of iterators: %d vs. %d\n",
|
|
|
|
step, miter->Valid(), dbiter->Valid());
|
|
|
|
ok = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fprintf(stderr, "%d entries compared: ok=%d\n", count, ok);
|
|
|
|
delete miter;
|
|
|
|
delete dbiter;
|
|
|
|
return ok;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST(DBTest, Randomized) {
|
|
|
|
Random rnd(test::RandomSeed());
|
2012-04-17 17:36:46 +02:00
|
|
|
do {
|
|
|
|
ModelDB model(CurrentOptions());
|
|
|
|
const int N = 10000;
|
|
|
|
const Snapshot* model_snap = NULL;
|
|
|
|
const Snapshot* db_snap = NULL;
|
|
|
|
std::string k, v;
|
|
|
|
for (int step = 0; step < N; step++) {
|
|
|
|
if (step % 100 == 0) {
|
|
|
|
fprintf(stderr, "Step %d of %d\n", step, N);
|
|
|
|
}
|
|
|
|
// TODO(sanjay): Test Get() works
|
|
|
|
int p = rnd.Uniform(100);
|
|
|
|
if (p < 45) { // Put
|
|
|
|
k = RandomKey(&rnd);
|
|
|
|
v = RandomString(&rnd,
|
|
|
|
rnd.OneIn(20)
|
|
|
|
? 100 + rnd.Uniform(100)
|
|
|
|
: rnd.Uniform(8));
|
|
|
|
ASSERT_OK(model.Put(WriteOptions(), k, v));
|
|
|
|
ASSERT_OK(db_->Put(WriteOptions(), k, v));
|
|
|
|
|
|
|
|
} else if (p < 90) { // Delete
|
|
|
|
k = RandomKey(&rnd);
|
|
|
|
ASSERT_OK(model.Delete(WriteOptions(), k));
|
|
|
|
ASSERT_OK(db_->Delete(WriteOptions(), k));
|
|
|
|
|
|
|
|
|
|
|
|
} else { // Multi-element batch
|
|
|
|
WriteBatch b;
|
|
|
|
const int num = rnd.Uniform(8);
|
|
|
|
for (int i = 0; i < num; i++) {
|
|
|
|
if (i == 0 || !rnd.OneIn(10)) {
|
|
|
|
k = RandomKey(&rnd);
|
|
|
|
} else {
|
|
|
|
// Periodically re-use the same key from the previous iter, so
|
|
|
|
// we have multiple entries in the write batch for the same key
|
|
|
|
}
|
|
|
|
if (rnd.OneIn(2)) {
|
|
|
|
v = RandomString(&rnd, rnd.Uniform(10));
|
|
|
|
b.Put(k, v);
|
|
|
|
} else {
|
|
|
|
b.Delete(k);
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
ASSERT_OK(model.Write(WriteOptions(), &b));
|
|
|
|
ASSERT_OK(db_->Write(WriteOptions(), &b));
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
if ((step % 100) == 0) {
|
|
|
|
ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
|
|
|
|
ASSERT_TRUE(CompareIterators(step, &model, db_, model_snap, db_snap));
|
|
|
|
// Save a snapshot from each DB this time that we'll use next
|
|
|
|
// time we compare things, to make sure the current state is
|
|
|
|
// preserved with the snapshot
|
|
|
|
if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
|
|
|
|
if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
Reopen();
|
|
|
|
ASSERT_TRUE(CompareIterators(step, &model, db_, NULL, NULL));
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2012-04-17 17:36:46 +02:00
|
|
|
model_snap = model.GetSnapshot();
|
|
|
|
db_snap = db_->GetSnapshot();
|
|
|
|
}
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
2012-04-17 17:36:46 +02:00
|
|
|
if (model_snap != NULL) model.ReleaseSnapshot(model_snap);
|
|
|
|
if (db_snap != NULL) db_->ReleaseSnapshot(db_snap);
|
|
|
|
} while (ChangeOptions());
|
2011-03-18 23:37:00 +01:00
|
|
|
}
|
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
std::string MakeKey(unsigned int num) {
|
|
|
|
char buf[30];
|
|
|
|
snprintf(buf, sizeof(buf), "%016u", num);
|
|
|
|
return std::string(buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
void BM_LogAndApply(int iters, int num_base_files) {
|
|
|
|
std::string dbname = test::TmpDir() + "/leveldb_test_benchmark";
|
|
|
|
DestroyDB(dbname, Options());
|
|
|
|
|
|
|
|
DB* db = NULL;
|
|
|
|
Options opts;
|
|
|
|
opts.create_if_missing = true;
|
|
|
|
Status s = DB::Open(opts, dbname, &db);
|
|
|
|
ASSERT_OK(s);
|
|
|
|
ASSERT_TRUE(db != NULL);
|
|
|
|
|
|
|
|
delete db;
|
|
|
|
db = NULL;
|
|
|
|
|
|
|
|
Env* env = Env::Default();
|
|
|
|
|
2011-09-01 21:08:02 +02:00
|
|
|
port::Mutex mu;
|
|
|
|
MutexLock l(&mu);
|
|
|
|
|
2011-05-21 04:17:43 +02:00
|
|
|
InternalKeyComparator cmp(BytewiseComparator());
|
|
|
|
Options options;
|
|
|
|
VersionSet vset(dbname, &options, NULL, &cmp);
|
|
|
|
ASSERT_OK(vset.Recover());
|
2012-06-23 04:30:03 +02:00
|
|
|
VersionEdit vbase(vset.NumberLevels());
|
2011-05-21 04:17:43 +02:00
|
|
|
uint64_t fnum = 1;
|
|
|
|
for (int i = 0; i < num_base_files; i++) {
|
|
|
|
InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
|
|
|
|
InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
|
|
|
|
vbase.AddFile(2, fnum++, 1 /* file size */, start, limit);
|
|
|
|
}
|
2011-09-01 21:08:02 +02:00
|
|
|
ASSERT_OK(vset.LogAndApply(&vbase, &mu));
|
2011-05-21 04:17:43 +02:00
|
|
|
|
|
|
|
uint64_t start_micros = env->NowMicros();
|
|
|
|
|
|
|
|
for (int i = 0; i < iters; i++) {
|
2012-06-23 04:30:03 +02:00
|
|
|
VersionEdit vedit(vset.NumberLevels());
|
2011-05-21 04:17:43 +02:00
|
|
|
vedit.DeleteFile(2, fnum);
|
|
|
|
InternalKey start(MakeKey(2*fnum), 1, kTypeValue);
|
|
|
|
InternalKey limit(MakeKey(2*fnum+1), 1, kTypeDeletion);
|
|
|
|
vedit.AddFile(2, fnum++, 1 /* file size */, start, limit);
|
2011-09-01 21:08:02 +02:00
|
|
|
vset.LogAndApply(&vedit, &mu);
|
2011-05-21 04:17:43 +02:00
|
|
|
}
|
|
|
|
uint64_t stop_micros = env->NowMicros();
|
|
|
|
unsigned int us = stop_micros - start_micros;
|
|
|
|
char buf[16];
|
|
|
|
snprintf(buf, sizeof(buf), "%d", num_base_files);
|
|
|
|
fprintf(stderr,
|
|
|
|
"BM_LogAndApply/%-6s %8d iters : %9u us (%7.0f us / iter)\n",
|
|
|
|
buf, iters, us, ((float)us) / iters);
|
|
|
|
}
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
} // namespace leveldb
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
2011-05-21 04:17:43 +02:00
|
|
|
if (argc > 1 && std::string(argv[1]) == "--benchmark") {
|
|
|
|
leveldb::BM_LogAndApply(1000, 1);
|
|
|
|
leveldb::BM_LogAndApply(1000, 100);
|
|
|
|
leveldb::BM_LogAndApply(1000, 10000);
|
|
|
|
leveldb::BM_LogAndApply(100, 100000);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
return leveldb::test::RunAllTests();
|
|
|
|
}
|