andreacavalli/rocksdb
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update blob_db_test

Browse Source 
Summary:
Re-enable blob_db_test with some update:
* Commented out delay at the end of GC tests. Will update the logic later with sync point to properly trigger GC.
* Added some helper functions.

Also update make files to include blob_dump tool.
Closes https://github.com/facebook/rocksdb/pull/2375

Differential Revision: D5133793

Pulled By: yiwu-arbug

fbshipit-source-id: 95470b26d0c1f9592ba4b7637e027fdd263f425c
This commit is contained in:
Yi Wu 2017-05-30 22:16:32 -07:00 committed by Facebook Github Bot
parent cbc821c25b
commit 345878a7fb
5 changed files with 109 additions and 173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -475,6 +475,7 @@ set(SOURCES
utilities/blob_db/blob_db.cc
utilities/blob_db/blob_db_impl.cc
utilities/blob_db/blob_db_options_impl.cc
utilities/blob_db/blob_dump_tool.cc
utilities/blob_db/blob_file.cc
utilities/blob_db/blob_log_reader.cc
utilities/blob_db/blob_log_writer.cc
@ -696,6 +697,7 @@ set(TESTS
util/thread_list_test.cc
util/thread_local_test.cc
utilities/backupable/backupable_db_test.cc
utilities/blob_db/blob_db_test.cc
utilities/checkpoint/checkpoint_test.cc
utilities/column_aware_encoding_test.cc
utilities/date_tiered/date_tiered_test.cc

3
Makefile
View File

@ -1377,6 +1377,9 @@ lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTH
range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(AM_LINK)
#-------------------------------------------------
# make install related stuff
INSTALL_PATH ?= /usr/local

5
TARGETS
View File

@ -200,6 +200,7 @@ cpp_library(
"utilities/blob_db/blob_db.cc",
"utilities/blob_db/blob_db_impl.cc",
"utilities/blob_db/blob_db_options_impl.cc",
"utilities/blob_db/blob_dump_tool.cc",
"utilities/blob_db/blob_file.cc",
"utilities/blob_db/blob_log_reader.cc",
"utilities/blob_db/blob_log_writer.cc",
@ -456,8 +457,8 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
['version_edit_test', 'db/version_edit_test.cc', 'serial'],
['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial']]
# Generate a test rule for each entry in ROCKS_TESTS
for test_cfg in ROCKS_TESTS:

1
src.mk
View File

@ -320,6 +320,7 @@ MAIN_SOURCES = \
util/thread_list_test.cc \
util/thread_local_test.cc \
utilities/backupable/backupable_db_test.cc \
utilities/blob_db/blob_db_test.cc \
utilities/checkpoint/checkpoint_test.cc \
utilities/column_aware_encoding_exp.cc \
utilities/column_aware_encoding_test.cc \

269
utilities/blob_db/blob_db_test.cc
View File

@ -7,10 +7,15 @@
#include "utilities/blob_db/blob_db.h"
#include <cstdlib>
#include <map>
#include <memory>
#include <string>
#include "db/db_test_util.h"
#include "port/port.h"
#include "util/random.h"
#include "util/string_util.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "utilities/blob_db/blob_db_impl.h"
#include "utilities/blob_db/blob_db_options_impl.h"
namespace rocksdb {
@ -33,9 +38,11 @@ void gen_random(char *s, const int len) {
class BlobDBTest : public testing::Test {
public:
const int kMaxBlobSize = 1 << 14;
BlobDBTest() : blobdb_(nullptr) {
dbname_ = test::TmpDir() + "/blob_db_test";
// Reopen1(BlobDBOptionsImpl());
// Reopen(BlobDBOptionsImpl());
}
~BlobDBTest() {
@ -45,7 +52,7 @@ class BlobDBTest : public testing::Test {
}
}
void Reopen1(const BlobDBOptionsImpl &bdboptions,
void Reopen(const BlobDBOptionsImpl &bdboptions,
const Options &options = Options()) {
if (blobdb_) {
delete blobdb_;
@ -61,40 +68,59 @@ class BlobDBTest : public testing::Test {
myoptions.create_if_missing = true;
EXPECT_TRUE(
BlobDB::Open(myoptions, bblobdb_options, dbname_, &blobdb_).ok());
ASSERT_NE(nullptr, blobdb_);
}
void insert_blobs() {
WriteOptions wo;
ReadOptions ro;
std::string value;
void PutRandomWithTTL(const std::string &key, int32_t ttl, Random *rnd,
std::map<std::string, std::string> *data = nullptr) {
int len = rnd->Next() % kMaxBlobSize + 1;
std::string value = test::RandomHumanReadableString(rnd, len);
ColumnFamilyHandle *cfh = blobdb_->DefaultColumnFamily();
ASSERT_OK(blobdb_->PutWithTTL(WriteOptions(), cfh, Slice(key), Slice(value),
ttl));
if (data != nullptr) {
(*data)[key] = value;
}
}
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
void PutRandom(const std::string &key, Random *rnd,
std::map<std::string, std::string> *data = nullptr) {
PutRandomWithTTL(key, -1, rnd, data);
}
void Delete(const std::string &key) {
ColumnFamilyHandle *cfh = blobdb_->DefaultColumnFamily();
ASSERT_OK(blobdb_->Delete(WriteOptions(), cfh, key));
}
// Verify blob db contain expected data and nothing more.
// TODO(yiwu): Verify blob files are consistent with data in LSM.
void VerifyDB(const std::map<std::string, std::string> &data) {
Iterator *iter = blobdb_->NewIterator(ReadOptions());
iter->SeekToFirst();
for (auto &p : data) {
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(p.first, iter->key().ToString());
ASSERT_EQ(p.second, iter->value().ToString());
iter->Next();
}
ASSERT_FALSE(iter->Valid());
ASSERT_OK(iter->status());
delete iter;
}
void InsertBlobs() {
WriteOptions wo;
std::string value;
Random rnd(301);
for (size_t i = 0; i < 100000; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
key += std::to_string(i % 500);
Slice keyslice(key);
Slice valslice(val, len + 1);
int ttl = rnd.Next() % 86400;
ASSERT_OK(blobdb_->PutWithTTL(wo, dcfh, keyslice, valslice, ttl));
delete[] val;
int32_t ttl = rnd.Next() % 86400;
PutRandomWithTTL("key" + ToString(i % 500), ttl, &rnd, nullptr);
}
for (size_t i = 0; i < 10; i++) {
std::string key("key");
key += std::to_string(i % 500);
Slice keyslice(key);
blobdb_->Delete(wo, dcfh, keyslice);
Delete("key" + ToString(i % 500));
}
}
@ -104,162 +130,60 @@ class BlobDBTest : public testing::Test {
TEST_F(BlobDBTest, DeleteComplex) {
BlobDBOptionsImpl bdboptions;
bdboptions.partial_expiration_pct = 75;
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.blob_file_size = 219 * 1024;
Reopen1(bdboptions);
WriteOptions wo;
ReadOptions ro;
std::string value;
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
Reopen(bdboptions);
Random rnd(301);
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, nullptr);
}
for (size_t i = 0; i < 99; i++) {
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
blobdb_->Delete(wo, dcfh, keyslice);
for (size_t i = 0; i < 100; i++) {
Delete("key" + ToString(i));
}
Env::Default()->SleepForMicroseconds(60 * 1000 * 1000);
// DB should be empty.
VerifyDB({});
}
TEST_F(BlobDBTest, OverrideTest) {
BlobDBOptionsImpl bdboptions;
bdboptions.ttl_range_secs = 30;
bdboptions.gc_file_pct = 100;
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.num_concurrent_simple_blobs = 2;
bdboptions.blob_file_size = 876 * 1024 * 10;
Options options;
options.write_buffer_size = 256 * 1024;
options.info_log_level = INFO_LEVEL;
Reopen1(bdboptions, options);
WriteOptions wo;
ReadOptions ro;
std::string value;
Reopen(bdboptions, options);
Random rnd(301);
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
std::map<std::string, std::string> data;
for (int i = 0; i < 10000; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
char x[10];
std::sprintf(x, "%04d", i);
key += std::string(x);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, nullptr);
}
// override all the keys
for (int i = 0; i < 10000; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
char x[10];
std::sprintf(x, "%04d", i);
key += std::string(x);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, &data);
}
blobdb_->Flush(FlushOptions());
#if 1
blobdb_->GetBaseDB()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
reinterpret_cast<DBImpl *>(blobdb_->GetBaseDB())->TEST_WaitForFlushMemTable();
reinterpret_cast<DBImpl *>(blobdb_->GetBaseDB())->TEST_WaitForCompact();
#endif
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
VerifyDB(data);
}
TEST_F(BlobDBTest, DeleteTest) {
BlobDBOptionsImpl bdboptions;
bdboptions.ttl_range_secs = 30;
bdboptions.gc_file_pct = 100;
bdboptions.partial_expiration_pct = 18;
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.num_concurrent_simple_blobs = 1;
bdboptions.blob_file_size = 876 * 1024;
Reopen1(bdboptions);
WriteOptions wo;
ReadOptions ro;
std::string value;
Reopen(bdboptions);
Random rnd(301);
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
std::map<std::string, std::string> data;
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
if (!len) continue;
char *val = new char[len + 1];
gen_random(val, len);
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
Slice valslice(val, len + 1);
ASSERT_OK(blobdb_->Put(wo, dcfh, keyslice, valslice));
delete[] val;
PutRandom("key" + ToString(i), &rnd, &data);
}
for (size_t i = 0; i < 100; i += 5) {
std::string key("key");
key += std::to_string(i);
Slice keyslice(key);
blobdb_->Delete(wo, dcfh, keyslice);
Delete("key" + ToString(i));
data.erase("key" + ToString(i));
}
Env::Default()->SleepForMicroseconds(60 * 1000 * 1000);
VerifyDB(data);
}
TEST_F(BlobDBTest, GCTestWithWrite) {
@ -269,7 +193,7 @@ TEST_F(BlobDBTest, GCTestWithWrite) {
bdboptions.gc_check_period_millisecs = 20 * 1000;
bdboptions.default_ttl_extractor = true;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -277,11 +201,11 @@ TEST_F(BlobDBTest, GCTestWithWrite) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
WriteBatch WB;
WriteBatch batch;
Random rnd(301);
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
int ttl = 30;
@ -297,13 +221,14 @@ TEST_F(BlobDBTest, GCTestWithWrite) {
Slice keyslice(key);
Slice valslice(val, len + BlobDB::kTTLSuffixLength);
WB.Put(dcfh, keyslice, valslice);
batch.Put(dcfh, keyslice, valslice);
delete[] val;
}
ASSERT_OK(blobdb_->Write(wo, &WB));
ASSERT_OK(blobdb_->Write(wo, &batch));
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
void cb_evict(const ColumnFamilyHandle *cfh, const Slice &key,
@ -348,7 +273,7 @@ TEST_F(BlobDBTest, GetWithCompression) {
bdboptions.gc_evict_cb_fn = &cb_evict;
bdboptions.compression = CompressionType::kLZ4Compression;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -360,7 +285,7 @@ TEST_F(BlobDBTest, GetWithCompression) {
std::string orig(LONG_STRING);
for (size_t i = 0; i < 10000; i++) {
int len = orig.length();
size_t len = orig.length();
int ttl = 3000 * (rnd.Next() % 10);
char *val = new char[len + BlobDB::kTTLSuffixLength];
@ -388,7 +313,8 @@ TEST_F(BlobDBTest, GetWithCompression) {
ASSERT_TRUE(orig == val);
}
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
@ -400,7 +326,7 @@ TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
bdboptions.gc_evict_cb_fn = &cb_evict;
bdboptions.compression = CompressionType::kLZ4Compression;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -410,7 +336,7 @@ TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
int ttl = 30;
@ -430,7 +356,8 @@ TEST_F(BlobDBTest, GCTestWithPutAndCompression) {
delete[] val;
}
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
TEST_F(BlobDBTest, GCTestWithPut) {
@ -441,7 +368,7 @@ TEST_F(BlobDBTest, GCTestWithPut) {
bdboptions.default_ttl_extractor = true;
bdboptions.gc_evict_cb_fn = &cb_evict;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -451,7 +378,7 @@ TEST_F(BlobDBTest, GCTestWithPut) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
int ttl = 30;
@ -471,7 +398,8 @@ TEST_F(BlobDBTest, GCTestWithPut) {
delete[] val;
}
Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(120 * 1000 * 1000);
}
TEST_F(BlobDBTest, GCTest) {
@ -479,7 +407,7 @@ TEST_F(BlobDBTest, GCTest) {
bdboptions.ttl_range_secs = 30;
bdboptions.gc_file_pct = 100;
Reopen1(bdboptions);
Reopen(bdboptions);
WriteOptions wo;
ReadOptions ro;
@ -489,7 +417,7 @@ TEST_F(BlobDBTest, GCTest) {
ColumnFamilyHandle *dcfh = blobdb_->DefaultColumnFamily();
for (size_t i = 0; i < 100; i++) {
int len = rnd.Next() % 16384;
int len = rnd.Next() % kMaxBlobSize;
if (!len) continue;
char *val = new char[len + 1];
@ -507,18 +435,19 @@ TEST_F(BlobDBTest, GCTest) {
delete[] val;
}
Env::Default()->SleepForMicroseconds(240 * 1000 * 1000);
// TODO(yiwu): Use sync point to properly trigger GC and check result.
// Env::Default()->SleepForMicroseconds(240 * 1000 * 1000);
}
TEST_F(BlobDBTest, DISABLED_MultipleWriters) {
BlobDBOptionsImpl bdboptions;
Reopen1(bdboptions);
Reopen(bdboptions);
ASSERT_TRUE(blobdb_ != nullptr);
std::vector<std::thread> workers;
for (size_t ii = 0; ii < 10; ii++)
workers.push_back(std::thread(&BlobDBTest::insert_blobs, this));
workers.push_back(std::thread(&BlobDBTest::InsertBlobs, this));
for (std::thread &t : workers) {
if (t.joinable()) {
@ -534,9 +463,10 @@ TEST_F(BlobDBTest, DISABLED_MultipleWriters) {
// ASSERT_EQ("v2", value);
}
#if 0
TEST_F(BlobDBTest, Large) {
ASSERT_TRUE(blobdb_ != nullptr);
BlobDBOptionsImpl bdboptions;
Options options;
Reopen(bdboptions, options);
WriteOptions wo;
ReadOptions ro;
@ -559,7 +489,6 @@ TEST_F(BlobDBTest, Large) {
ASSERT_OK(blobdb_->Get(ro, dcfh, "barfoo", &value));
ASSERT_EQ(value3, value);
}
#endif
} // namespace blob_db
} // namespace rocksdb