2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2015-09-01 08:11:12 +02:00
|
|
|
|
2020-03-12 18:58:27 +01:00
|
|
|
#include "db/blob/blob_index.h"
|
2019-06-01 00:21:36 +02:00
|
|
|
#include "db/db_impl/db_impl.h"
|
2015-12-14 22:36:32 +01:00
|
|
|
#include "db/db_test_util.h"
|
2015-09-01 08:11:12 +02:00
|
|
|
#include "db/dbformat.h"
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
#include "db/version_set.h"
|
|
|
|
#include "db/write_batch_internal.h"
|
2019-05-30 05:44:08 +02:00
|
|
|
#include "file/filename.h"
|
2015-10-16 23:10:33 +02:00
|
|
|
#include "memtable/hash_linklist_rep.h"
|
2017-04-06 04:02:00 +02:00
|
|
|
#include "monitoring/statistics.h"
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
#include "rocksdb/cache.h"
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/env.h"
|
|
|
|
#include "rocksdb/filter_policy.h"
|
2015-09-01 08:11:12 +02:00
|
|
|
#include "rocksdb/options.h"
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
#include "rocksdb/perf_context.h"
|
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
#include "rocksdb/slice_transform.h"
|
|
|
|
#include "rocksdb/table.h"
|
|
|
|
#include "rocksdb/table_properties.h"
|
2019-05-31 02:39:43 +02:00
|
|
|
#include "test_util/sync_point.h"
|
|
|
|
#include "test_util/testharness.h"
|
|
|
|
#include "test_util/testutil.h"
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
#include "util/hash.h"
|
|
|
|
#include "util/mutexlock.h"
|
|
|
|
#include "util/rate_limiter.h"
|
2015-03-20 01:29:37 +01:00
|
|
|
#include "util/string_util.h"
|
2015-09-01 08:11:12 +02:00
|
|
|
#include "utilities/merge_operators.h"
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
2015-12-14 22:36:32 +01:00
|
|
|
class EventListenerTest : public DBTestBase {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
public:
|
2020-08-18 03:41:20 +02:00
|
|
|
EventListenerTest() : DBTestBase("/listener_test", /*env_do_fsync=*/true) {}
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
2019-10-24 23:42:43 +02:00
|
|
|
static std::string BlobStr(uint64_t blob_file_number, uint64_t offset,
|
|
|
|
uint64_t size) {
|
|
|
|
std::string blob_index;
|
|
|
|
BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size,
|
|
|
|
kNoCompression);
|
|
|
|
return blob_index;
|
|
|
|
}
|
|
|
|
|
2015-09-01 08:11:12 +02:00
|
|
|
const size_t k110KB = 110 << 10;
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
};
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
struct TestPropertiesCollector
|
|
|
|
: public ROCKSDB_NAMESPACE::TablePropertiesCollector {
|
|
|
|
ROCKSDB_NAMESPACE::Status AddUserKey(
|
|
|
|
const ROCKSDB_NAMESPACE::Slice& /*key*/,
|
|
|
|
const ROCKSDB_NAMESPACE::Slice& /*value*/,
|
|
|
|
ROCKSDB_NAMESPACE::EntryType /*type*/,
|
|
|
|
ROCKSDB_NAMESPACE::SequenceNumber /*seq*/,
|
|
|
|
uint64_t /*file_size*/) override {
|
2015-09-15 18:03:08 +02:00
|
|
|
return Status::OK();
|
|
|
|
}
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::Status Finish(
|
|
|
|
ROCKSDB_NAMESPACE::UserCollectedProperties* properties) override {
|
2015-09-15 18:03:08 +02:00
|
|
|
properties->insert({"0", "1"});
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2019-02-14 22:52:47 +01:00
|
|
|
const char* Name() const override { return "TestTablePropertiesCollector"; }
|
2015-09-15 18:03:08 +02:00
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::UserCollectedProperties GetReadableProperties()
|
|
|
|
const override {
|
|
|
|
ROCKSDB_NAMESPACE::UserCollectedProperties ret;
|
2015-09-15 18:03:08 +02:00
|
|
|
ret["2"] = "3";
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
class TestPropertiesCollectorFactory : public TablePropertiesCollectorFactory {
|
|
|
|
public:
|
2019-02-14 22:52:47 +01:00
|
|
|
TablePropertiesCollector* CreateTablePropertiesCollector(
|
2018-03-05 22:08:17 +01:00
|
|
|
TablePropertiesCollectorFactory::Context /*context*/) override {
|
2015-09-15 18:03:08 +02:00
|
|
|
return new TestPropertiesCollector;
|
|
|
|
}
|
|
|
|
const char* Name() const override { return "TestTablePropertiesCollector"; }
|
|
|
|
};
|
|
|
|
|
2015-01-27 23:44:02 +01:00
|
|
|
class TestCompactionListener : public EventListener {
|
|
|
|
public:
|
2019-10-24 23:42:43 +02:00
|
|
|
explicit TestCompactionListener(EventListenerTest* test) : test_(test) {}
|
|
|
|
|
2015-01-28 00:01:04 +01:00
|
|
|
void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) override {
|
2015-05-13 01:10:23 +02:00
|
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
2015-01-27 23:44:02 +01:00
|
|
|
compacted_dbs_.push_back(db);
|
2015-05-13 01:10:23 +02:00
|
|
|
ASSERT_GT(ci.input_files.size(), 0U);
|
2019-10-24 23:42:43 +02:00
|
|
|
ASSERT_EQ(ci.input_files.size(), ci.input_file_infos.size());
|
|
|
|
|
|
|
|
for (size_t i = 0; i < ci.input_file_infos.size(); ++i) {
|
|
|
|
ASSERT_EQ(ci.input_file_infos[i].level, ci.base_input_level);
|
|
|
|
ASSERT_EQ(ci.input_file_infos[i].file_number,
|
|
|
|
TableFileNameToNumber(ci.input_files[i]));
|
|
|
|
}
|
|
|
|
|
2015-05-13 01:10:23 +02:00
|
|
|
ASSERT_GT(ci.output_files.size(), 0U);
|
2019-10-24 23:42:43 +02:00
|
|
|
ASSERT_EQ(ci.output_files.size(), ci.output_file_infos.size());
|
|
|
|
|
|
|
|
ASSERT_TRUE(test_);
|
|
|
|
ASSERT_EQ(test_->db_, db);
|
|
|
|
|
|
|
|
std::vector<std::vector<FileMetaData>> files_by_level;
|
|
|
|
test_->dbfull()->TEST_GetFilesMetaData(test_->handles_[ci.cf_id],
|
|
|
|
&files_by_level);
|
|
|
|
ASSERT_GT(files_by_level.size(), ci.output_level);
|
|
|
|
|
|
|
|
for (size_t i = 0; i < ci.output_file_infos.size(); ++i) {
|
|
|
|
ASSERT_EQ(ci.output_file_infos[i].level, ci.output_level);
|
|
|
|
ASSERT_EQ(ci.output_file_infos[i].file_number,
|
|
|
|
TableFileNameToNumber(ci.output_files[i]));
|
|
|
|
|
|
|
|
auto it = std::find_if(
|
|
|
|
files_by_level[ci.output_level].begin(),
|
|
|
|
files_by_level[ci.output_level].end(), [&](const FileMetaData& meta) {
|
|
|
|
return meta.fd.GetNumber() == ci.output_file_infos[i].file_number;
|
|
|
|
});
|
|
|
|
ASSERT_NE(it, files_by_level[ci.output_level].end());
|
|
|
|
|
|
|
|
ASSERT_EQ(ci.output_file_infos[i].oldest_blob_file_number,
|
|
|
|
it->oldest_blob_file_number);
|
|
|
|
}
|
|
|
|
|
2015-06-11 23:18:02 +02:00
|
|
|
ASSERT_EQ(db->GetEnv()->GetThreadID(), ci.thread_id);
|
|
|
|
ASSERT_GT(ci.thread_id, 0U);
|
2015-09-15 18:03:08 +02:00
|
|
|
|
|
|
|
for (auto fl : {ci.input_files, ci.output_files}) {
|
|
|
|
for (auto fn : fl) {
|
|
|
|
auto it = ci.table_properties.find(fn);
|
|
|
|
ASSERT_NE(it, ci.table_properties.end());
|
|
|
|
auto tp = it->second;
|
|
|
|
ASSERT_TRUE(tp != nullptr);
|
|
|
|
ASSERT_EQ(tp->user_collected_properties.find("0")->second, "1");
|
|
|
|
}
|
|
|
|
}
|
2015-01-27 23:44:02 +01:00
|
|
|
}
|
|
|
|
|
2019-10-24 23:42:43 +02:00
|
|
|
EventListenerTest* test_;
|
2015-01-27 23:44:02 +01:00
|
|
|
std::vector<DB*> compacted_dbs_;
|
2015-05-13 01:10:23 +02:00
|
|
|
std::mutex mutex_;
|
2015-01-27 23:44:02 +01:00
|
|
|
};
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
|
2015-01-27 23:44:02 +01:00
|
|
|
const int kTestKeySize = 16;
|
|
|
|
const int kTestValueSize = 984;
|
|
|
|
const int kEntrySize = kTestKeySize + kTestValueSize;
|
|
|
|
const int kEntriesPerBuffer = 100;
|
|
|
|
const int kNumL0Files = 4;
|
|
|
|
|
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2015-01-27 23:44:02 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
|
|
|
|
options.compaction_style = kCompactionStyleLevel;
|
|
|
|
options.target_file_size_base = options.write_buffer_size;
|
|
|
|
options.max_bytes_for_level_base = options.target_file_size_base * 2;
|
|
|
|
options.max_bytes_for_level_multiplier = 2;
|
|
|
|
options.compression = kNoCompression;
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2015-01-27 23:44:02 +01:00
|
|
|
options.enable_thread_tracking = true;
|
2015-06-11 23:18:02 +02:00
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2015-01-27 23:44:02 +01:00
|
|
|
options.level0_file_num_compaction_trigger = kNumL0Files;
|
2015-09-15 18:03:08 +02:00
|
|
|
options.table_properties_collector_factories.push_back(
|
|
|
|
std::make_shared<TestPropertiesCollectorFactory>());
|
2015-01-27 23:44:02 +01:00
|
|
|
|
2019-10-24 23:42:43 +02:00
|
|
|
TestCompactionListener* listener = new TestCompactionListener(this);
|
2015-01-27 23:44:02 +01:00
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
std::vector<std::string> cf_names = {
|
|
|
|
"pikachu", "ilya", "muromec", "dobrynia",
|
|
|
|
"nikitich", "alyosha", "popovich"};
|
2015-12-14 22:36:32 +01:00
|
|
|
CreateAndReopenWithCF(cf_names, options);
|
2015-01-27 23:44:02 +01:00
|
|
|
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
2019-10-24 23:42:43 +02:00
|
|
|
|
|
|
|
WriteBatch batch;
|
|
|
|
ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 1, "ditto",
|
|
|
|
BlobStr(123, 0, 1 << 10)));
|
|
|
|
ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
|
|
|
|
|
2015-01-27 23:44:02 +01:00
|
|
|
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
|
|
|
|
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
|
|
|
|
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
|
|
|
|
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
|
|
|
|
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
|
|
|
|
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
|
2015-12-14 22:36:32 +01:00
|
|
|
for (int i = 1; i < 8; ++i) {
|
2015-09-01 08:11:12 +02:00
|
|
|
ASSERT_OK(Flush(i));
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
2019-10-24 23:42:43 +02:00
|
|
|
ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i],
|
|
|
|
nullptr, nullptr));
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-01-27 23:44:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
ASSERT_EQ(listener->compacted_dbs_.size(), cf_names.size());
|
|
|
|
for (size_t i = 0; i < cf_names.size(); ++i) {
|
|
|
|
ASSERT_EQ(listener->compacted_dbs_[i], db_);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-02 23:12:23 +02:00
|
|
|
// This simple Listener can only handle one flush at a time.
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
class TestFlushListener : public EventListener {
|
|
|
|
public:
|
2019-10-24 23:42:43 +02:00
|
|
|
TestFlushListener(Env* env, EventListenerTest* test)
|
|
|
|
: slowdown_count(0), stop_count(0), db_closed(), env_(env), test_(test) {
|
2015-07-02 01:13:49 +02:00
|
|
|
db_closed = false;
|
|
|
|
}
|
2020-12-23 08:44:44 +01:00
|
|
|
|
|
|
|
virtual ~TestFlushListener() {
|
|
|
|
prev_fc_info_.status.PermitUncheckedError(); // Ignore the status
|
|
|
|
}
|
2015-06-02 23:12:23 +02:00
|
|
|
void OnTableFileCreated(
|
2015-06-02 23:20:27 +02:00
|
|
|
const TableFileCreationInfo& info) override {
|
2015-06-05 21:28:51 +02:00
|
|
|
// remember the info for later checking the FlushJobInfo.
|
|
|
|
prev_fc_info_ = info;
|
|
|
|
ASSERT_GT(info.db_name.size(), 0U);
|
|
|
|
ASSERT_GT(info.cf_name.size(), 0U);
|
|
|
|
ASSERT_GT(info.file_path.size(), 0U);
|
|
|
|
ASSERT_GT(info.job_id, 0);
|
2015-06-02 23:12:23 +02:00
|
|
|
ASSERT_GT(info.table_properties.data_size, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.raw_key_size, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.raw_value_size, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.num_data_blocks, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.num_entries, 0U);
|
2020-08-25 19:44:39 +02:00
|
|
|
ASSERT_EQ(info.file_checksum, kUnknownFileChecksum);
|
|
|
|
ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName);
|
2015-06-11 23:18:02 +02:00
|
|
|
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2015-06-11 23:18:02 +02:00
|
|
|
// Verify the id of the current thread that created this table
|
|
|
|
// file matches the id of any active flush or compaction thread.
|
|
|
|
uint64_t thread_id = env_->GetThreadID();
|
|
|
|
std::vector<ThreadStatus> thread_list;
|
|
|
|
ASSERT_OK(env_->GetThreadList(&thread_list));
|
|
|
|
bool found_match = false;
|
|
|
|
for (auto thread_status : thread_list) {
|
|
|
|
if (thread_status.operation_type == ThreadStatus::OP_FLUSH ||
|
|
|
|
thread_status.operation_type == ThreadStatus::OP_COMPACTION) {
|
|
|
|
if (thread_id == thread_status.thread_id) {
|
|
|
|
found_match = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ASSERT_TRUE(found_match);
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2015-06-02 23:12:23 +02:00
|
|
|
}
|
|
|
|
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
void OnFlushCompleted(
|
2015-06-05 21:28:51 +02:00
|
|
|
DB* db, const FlushJobInfo& info) override {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
flushed_dbs_.push_back(db);
|
2015-06-05 21:28:51 +02:00
|
|
|
flushed_column_family_names_.push_back(info.cf_name);
|
|
|
|
if (info.triggered_writes_slowdown) {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
slowdown_count++;
|
|
|
|
}
|
2015-06-05 21:28:51 +02:00
|
|
|
if (info.triggered_writes_stop) {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
stop_count++;
|
|
|
|
}
|
2015-06-05 21:28:51 +02:00
|
|
|
// verify whether the previously created file matches the flushed file.
|
|
|
|
ASSERT_EQ(prev_fc_info_.db_name, db->GetName());
|
|
|
|
ASSERT_EQ(prev_fc_info_.cf_name, info.cf_name);
|
|
|
|
ASSERT_EQ(prev_fc_info_.job_id, info.job_id);
|
|
|
|
ASSERT_EQ(prev_fc_info_.file_path, info.file_path);
|
2019-10-24 23:42:43 +02:00
|
|
|
ASSERT_EQ(TableFileNameToNumber(info.file_path), info.file_number);
|
|
|
|
|
|
|
|
// Note: the following chunk relies on the notification pertaining to the
|
|
|
|
// database pointed to by DBTestBase::db_, and is thus bypassed when
|
|
|
|
// that assumption does not hold (see the test case MultiDBMultiListeners
|
|
|
|
// below).
|
|
|
|
ASSERT_TRUE(test_);
|
|
|
|
if (db == test_->db_) {
|
|
|
|
std::vector<std::vector<FileMetaData>> files_by_level;
|
|
|
|
test_->dbfull()->TEST_GetFilesMetaData(test_->handles_[info.cf_id],
|
|
|
|
&files_by_level);
|
|
|
|
|
|
|
|
ASSERT_FALSE(files_by_level.empty());
|
|
|
|
auto it = std::find_if(files_by_level[0].begin(), files_by_level[0].end(),
|
|
|
|
[&](const FileMetaData& meta) {
|
|
|
|
return meta.fd.GetNumber() == info.file_number;
|
|
|
|
});
|
|
|
|
ASSERT_NE(it, files_by_level[0].end());
|
|
|
|
ASSERT_EQ(info.oldest_blob_file_number, it->oldest_blob_file_number);
|
|
|
|
}
|
|
|
|
|
2015-06-11 23:18:02 +02:00
|
|
|
ASSERT_EQ(db->GetEnv()->GetThreadID(), info.thread_id);
|
|
|
|
ASSERT_GT(info.thread_id, 0U);
|
2015-09-15 18:03:08 +02:00
|
|
|
ASSERT_EQ(info.table_properties.user_collected_properties.find("0")->second,
|
|
|
|
"1");
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::string> flushed_column_family_names_;
|
|
|
|
std::vector<DB*> flushed_dbs_;
|
|
|
|
int slowdown_count;
|
|
|
|
int stop_count;
|
2015-06-11 23:18:02 +02:00
|
|
|
bool db_closing;
|
|
|
|
std::atomic_bool db_closed;
|
2015-06-05 21:28:51 +02:00
|
|
|
TableFileCreationInfo prev_fc_info_;
|
2015-06-11 23:18:02 +02:00
|
|
|
|
|
|
|
protected:
|
|
|
|
Env* env_;
|
2019-10-24 23:42:43 +02:00
|
|
|
EventListenerTest* test_;
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
};
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(EventListenerTest, OnSingleDBFlushTest) {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2015-09-01 08:11:12 +02:00
|
|
|
options.write_buffer_size = k110KB;
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2015-06-11 23:18:02 +02:00
|
|
|
options.enable_thread_tracking = true;
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2019-10-24 23:42:43 +02:00
|
|
|
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
std::vector<std::string> cf_names = {
|
|
|
|
"pikachu", "ilya", "muromec", "dobrynia",
|
|
|
|
"nikitich", "alyosha", "popovich"};
|
2015-09-15 18:03:08 +02:00
|
|
|
options.table_properties_collector_factories.push_back(
|
|
|
|
std::make_shared<TestPropertiesCollectorFactory>());
|
2015-12-14 22:36:32 +01:00
|
|
|
CreateAndReopenWithCF(cf_names, options);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
2014-11-25 03:28:06 +01:00
|
|
|
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
2019-10-24 23:42:43 +02:00
|
|
|
|
|
|
|
WriteBatch batch;
|
|
|
|
ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 1, "ditto",
|
|
|
|
BlobStr(456, 0, 1 << 10)));
|
|
|
|
ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
|
|
|
|
|
2014-11-25 03:28:06 +01:00
|
|
|
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
|
|
|
|
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
|
|
|
|
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
|
|
|
|
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
|
|
|
|
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
|
|
|
|
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
|
2015-12-14 22:36:32 +01:00
|
|
|
for (int i = 1; i < 8; ++i) {
|
2015-09-01 08:11:12 +02:00
|
|
|
ASSERT_OK(Flush(i));
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
ASSERT_EQ(listener->flushed_dbs_.size(), i);
|
|
|
|
ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
|
|
|
|
}
|
|
|
|
|
2018-04-11 00:47:54 +02:00
|
|
|
// make sure callback functions are called in the right order
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
for (size_t i = 0; i < cf_names.size(); ++i) {
|
|
|
|
ASSERT_EQ(listener->flushed_dbs_[i], db_);
|
|
|
|
ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(EventListenerTest, MultiCF) {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2015-09-01 08:11:12 +02:00
|
|
|
options.write_buffer_size = k110KB;
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2015-06-11 23:18:02 +02:00
|
|
|
options.enable_thread_tracking = true;
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2019-10-24 23:42:43 +02:00
|
|
|
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
options.listeners.emplace_back(listener);
|
2015-09-15 18:03:08 +02:00
|
|
|
options.table_properties_collector_factories.push_back(
|
|
|
|
std::make_shared<TestPropertiesCollectorFactory>());
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
std::vector<std::string> cf_names = {
|
|
|
|
"pikachu", "ilya", "muromec", "dobrynia",
|
|
|
|
"nikitich", "alyosha", "popovich"};
|
2015-12-14 22:36:32 +01:00
|
|
|
CreateAndReopenWithCF(cf_names, options);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
2014-11-25 03:28:06 +01:00
|
|
|
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
|
|
|
|
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
|
|
|
|
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
|
|
|
|
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
|
|
|
|
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
|
|
|
|
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
|
|
|
|
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
|
2015-12-14 22:36:32 +01:00
|
|
|
for (int i = 1; i < 8; ++i) {
|
2015-09-01 08:11:12 +02:00
|
|
|
ASSERT_OK(Flush(i));
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
ASSERT_EQ(listener->flushed_dbs_.size(), i);
|
|
|
|
ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
|
|
|
|
}
|
|
|
|
|
2018-04-11 00:47:54 +02:00
|
|
|
// make sure callback functions are called in the right order
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
for (size_t i = 0; i < cf_names.size(); i++) {
|
|
|
|
ASSERT_EQ(listener->flushed_dbs_[i], db_);
|
|
|
|
ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(EventListenerTest, MultiDBMultiListeners) {
|
2015-06-11 23:18:02 +02:00
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2015-06-11 23:18:02 +02:00
|
|
|
options.enable_thread_tracking = true;
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2015-09-15 18:03:08 +02:00
|
|
|
options.table_properties_collector_factories.push_back(
|
|
|
|
std::make_shared<TestPropertiesCollectorFactory>());
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
std::vector<TestFlushListener*> listeners;
|
|
|
|
const int kNumDBs = 5;
|
|
|
|
const int kNumListeners = 10;
|
|
|
|
for (int i = 0; i < kNumListeners; ++i) {
|
2019-10-24 23:42:43 +02:00
|
|
|
listeners.emplace_back(new TestFlushListener(options.env, this));
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<std::string> cf_names = {
|
|
|
|
"pikachu", "ilya", "muromec", "dobrynia",
|
|
|
|
"nikitich", "alyosha", "popovich"};
|
|
|
|
|
|
|
|
options.create_if_missing = true;
|
|
|
|
for (int i = 0; i < kNumListeners; ++i) {
|
|
|
|
options.listeners.emplace_back(listeners[i]);
|
|
|
|
}
|
|
|
|
DBOptions db_opts(options);
|
|
|
|
ColumnFamilyOptions cf_opts(options);
|
|
|
|
|
|
|
|
std::vector<DB*> dbs;
|
|
|
|
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
|
|
|
|
|
|
|
|
for (int d = 0; d < kNumDBs; ++d) {
|
2014-11-25 05:44:49 +01:00
|
|
|
ASSERT_OK(DestroyDB(dbname_ + ToString(d), options));
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
DB* db;
|
|
|
|
std::vector<ColumnFamilyHandle*> handles;
|
2014-11-25 05:44:49 +01:00
|
|
|
ASSERT_OK(DB::Open(options, dbname_ + ToString(d), &db));
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
|
|
|
ColumnFamilyHandle* handle;
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(db->CreateColumnFamily(cf_opts, cf_names[c], &handle));
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
handles.push_back(handle);
|
|
|
|
}
|
|
|
|
|
|
|
|
vec_handles.push_back(std::move(handles));
|
|
|
|
dbs.push_back(db);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (int d = 0; d < kNumDBs; ++d) {
|
|
|
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
|
|
|
ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c],
|
|
|
|
cf_names[c], cf_names[c]));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
|
|
|
for (int d = 0; d < kNumDBs; ++d) {
|
|
|
|
ASSERT_OK(dbs[d]->Flush(FlushOptions(), vec_handles[d][c]));
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(
|
|
|
|
static_cast_with_check<DBImpl>(dbs[d])->TEST_WaitForFlushMemTable());
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (auto* listener : listeners) {
|
|
|
|
int pos = 0;
|
|
|
|
for (size_t c = 0; c < cf_names.size(); ++c) {
|
|
|
|
for (int d = 0; d < kNumDBs; ++d) {
|
|
|
|
ASSERT_EQ(listener->flushed_dbs_[pos], dbs[d]);
|
|
|
|
ASSERT_EQ(listener->flushed_column_family_names_[pos], cf_names[c]);
|
|
|
|
pos++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-11 23:18:02 +02:00
|
|
|
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
for (auto handles : vec_handles) {
|
|
|
|
for (auto h : handles) {
|
|
|
|
delete h;
|
|
|
|
}
|
|
|
|
handles.clear();
|
|
|
|
}
|
|
|
|
vec_handles.clear();
|
|
|
|
|
|
|
|
for (auto db : dbs) {
|
|
|
|
delete db;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-17 22:08:00 +01:00
|
|
|
TEST_F(EventListenerTest, DisableBGCompaction) {
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2016-12-14 03:22:00 +01:00
|
|
|
#ifdef ROCKSDB_USING_THREAD_STATUS
|
2015-06-11 23:18:02 +02:00
|
|
|
options.enable_thread_tracking = true;
|
|
|
|
#endif // ROCKSDB_USING_THREAD_STATUS
|
2019-10-24 23:42:43 +02:00
|
|
|
TestFlushListener* listener = new TestFlushListener(options.env, this);
|
Deprecate WriteOptions::timeout_hint_us
Summary:
In one of our recent meetings, we discussed deprecating features that are not being actively used. One of those features, at least within Facebook, is timeout_hint. The feature is really nicely implemented, but if nobody needs it, we should remove it from our code-base (until we get a valid use-case). Some arguments:
* Less code == better icache hit rate, smaller builds, simpler code
* The motivation for adding timeout_hint_us was to work-around RocksDB's stall issue. However, we're currently addressing the stall issue itself (see @sdong's recent work on stall write_rate), so we should never see sharp lock-ups in the future.
* Nobody is using the feature within Facebook's code-base. Googling for `timeout_hint_us` also doesn't yield any users.
Test Plan: make check
Reviewers: anthony, kradhakrishnan, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D41937
2015-07-14 09:35:48 +02:00
|
|
|
const int kCompactionTrigger = 1;
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
const int kSlowdownTrigger = 5;
|
Deprecate WriteOptions::timeout_hint_us
Summary:
In one of our recent meetings, we discussed deprecating features that are not being actively used. One of those features, at least within Facebook, is timeout_hint. The feature is really nicely implemented, but if nobody needs it, we should remove it from our code-base (until we get a valid use-case). Some arguments:
* Less code == better icache hit rate, smaller builds, simpler code
* The motivation for adding timeout_hint_us was to work-around RocksDB's stall issue. However, we're currently addressing the stall issue itself (see @sdong's recent work on stall write_rate), so we should never see sharp lock-ups in the future.
* Nobody is using the feature within Facebook's code-base. Googling for `timeout_hint_us` also doesn't yield any users.
Test Plan: make check
Reviewers: anthony, kradhakrishnan, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D41937
2015-07-14 09:35:48 +02:00
|
|
|
const int kStopTrigger = 100;
|
|
|
|
options.level0_file_num_compaction_trigger = kCompactionTrigger;
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
options.level0_slowdown_writes_trigger = kSlowdownTrigger;
|
|
|
|
options.level0_stop_writes_trigger = kStopTrigger;
|
Deprecate WriteOptions::timeout_hint_us
Summary:
In one of our recent meetings, we discussed deprecating features that are not being actively used. One of those features, at least within Facebook, is timeout_hint. The feature is really nicely implemented, but if nobody needs it, we should remove it from our code-base (until we get a valid use-case). Some arguments:
* Less code == better icache hit rate, smaller builds, simpler code
* The motivation for adding timeout_hint_us was to work-around RocksDB's stall issue. However, we're currently addressing the stall issue itself (see @sdong's recent work on stall write_rate), so we should never see sharp lock-ups in the future.
* Nobody is using the feature within Facebook's code-base. Googling for `timeout_hint_us` also doesn't yield any users.
Test Plan: make check
Reviewers: anthony, kradhakrishnan, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D41937
2015-07-14 09:35:48 +02:00
|
|
|
options.max_write_buffer_number = 10;
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
// BG compaction is disabled. Number of L0 files will simply keeps
|
|
|
|
// increasing in this test.
|
|
|
|
options.compaction_style = kCompactionStyleNone;
|
|
|
|
options.compression = kNoCompression;
|
|
|
|
options.write_buffer_size = 100000; // Small write buffer
|
2015-09-15 18:03:08 +02:00
|
|
|
options.table_properties_collector_factories.push_back(
|
|
|
|
std::make_shared<TestPropertiesCollectorFactory>());
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
2015-12-14 22:36:32 +01:00
|
|
|
CreateAndReopenWithCF({"pikachu"}, options);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
ColumnFamilyMetaData cf_meta;
|
|
|
|
db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
Deprecate WriteOptions::timeout_hint_us
Summary:
In one of our recent meetings, we discussed deprecating features that are not being actively used. One of those features, at least within Facebook, is timeout_hint. The feature is really nicely implemented, but if nobody needs it, we should remove it from our code-base (until we get a valid use-case). Some arguments:
* Less code == better icache hit rate, smaller builds, simpler code
* The motivation for adding timeout_hint_us was to work-around RocksDB's stall issue. However, we're currently addressing the stall issue itself (see @sdong's recent work on stall write_rate), so we should never see sharp lock-ups in the future.
* Nobody is using the feature within Facebook's code-base. Googling for `timeout_hint_us` also doesn't yield any users.
Test Plan: make check
Reviewers: anthony, kradhakrishnan, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D41937
2015-07-14 09:35:48 +02:00
|
|
|
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
// keep writing until writes are forced to stop.
|
Deprecate WriteOptions::timeout_hint_us
Summary:
In one of our recent meetings, we discussed deprecating features that are not being actively used. One of those features, at least within Facebook, is timeout_hint. The feature is really nicely implemented, but if nobody needs it, we should remove it from our code-base (until we get a valid use-case). Some arguments:
* Less code == better icache hit rate, smaller builds, simpler code
* The motivation for adding timeout_hint_us was to work-around RocksDB's stall issue. However, we're currently addressing the stall issue itself (see @sdong's recent work on stall write_rate), so we should never see sharp lock-ups in the future.
* Nobody is using the feature within Facebook's code-base. Googling for `timeout_hint_us` also doesn't yield any users.
Test Plan: make check
Reviewers: anthony, kradhakrishnan, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D41937
2015-07-14 09:35:48 +02:00
|
|
|
for (int i = 0; static_cast<int>(cf_meta.file_count) < kSlowdownTrigger * 10;
|
|
|
|
++i) {
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(Put(1, ToString(i), std::string(10000, 'x'), WriteOptions()));
|
2018-08-29 20:58:13 +02:00
|
|
|
FlushOptions fo;
|
|
|
|
fo.allow_write_stall = true;
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(db_->Flush(fo, handles_[1]));
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
|
|
|
|
}
|
Deprecate WriteOptions::timeout_hint_us
Summary:
In one of our recent meetings, we discussed deprecating features that are not being actively used. One of those features, at least within Facebook, is timeout_hint. The feature is really nicely implemented, but if nobody needs it, we should remove it from our code-base (until we get a valid use-case). Some arguments:
* Less code == better icache hit rate, smaller builds, simpler code
* The motivation for adding timeout_hint_us was to work-around RocksDB's stall issue. However, we're currently addressing the stall issue itself (see @sdong's recent work on stall write_rate), so we should never see sharp lock-ups in the future.
* Nobody is using the feature within Facebook's code-base. Googling for `timeout_hint_us` also doesn't yield any users.
Test Plan: make check
Reviewers: anthony, kradhakrishnan, sdong, yhchiang
Reviewed By: yhchiang
Subscribers: sdong, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D41937
2015-07-14 09:35:48 +02:00
|
|
|
ASSERT_GE(listener->slowdown_count, kSlowdownTrigger * 9);
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
}
|
|
|
|
|
2015-12-22 20:37:19 +01:00
|
|
|
class TestCompactionReasonListener : public EventListener {
|
|
|
|
public:
|
2018-03-05 22:08:17 +01:00
|
|
|
void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
|
2015-12-22 20:37:19 +01:00
|
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
|
|
compaction_reasons_.push_back(ci.compaction_reason);
|
|
|
|
}
|
|
|
|
|
|
|
|
std::vector<CompactionReason> compaction_reasons_;
|
|
|
|
std::mutex mutex_;
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, CompactionReasonLevel) {
|
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2015-12-22 20:37:19 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.memtable_factory.reset(
|
|
|
|
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
|
|
|
|
|
|
|
|
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
|
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
|
|
|
|
options.level0_file_num_compaction_trigger = 4;
|
|
|
|
options.compaction_style = kCompactionStyleLevel;
|
|
|
|
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Write 4 files in L0
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
|
|
GenerateNewRandomFile(&rnd);
|
|
|
|
}
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-12-22 20:37:19 +01:00
|
|
|
|
|
|
|
ASSERT_EQ(listener->compaction_reasons_.size(), 1);
|
|
|
|
ASSERT_EQ(listener->compaction_reasons_[0],
|
|
|
|
CompactionReason::kLevelL0FilesNum);
|
|
|
|
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
|
|
|
|
// Write 3 non-overlapping files in L0
|
|
|
|
for (int k = 1; k <= 30; k++) {
|
|
|
|
ASSERT_OK(Put(Key(k), Key(k)));
|
|
|
|
if (k % 10 == 0) {
|
|
|
|
Flush();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Do a trivial move from L0 -> L1
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
2015-12-22 20:37:19 +01:00
|
|
|
|
|
|
|
options.max_bytes_for_level_base = 1;
|
|
|
|
Close();
|
|
|
|
listener->compaction_reasons_.clear();
|
|
|
|
Reopen(options);
|
|
|
|
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-12-22 20:37:19 +01:00
|
|
|
ASSERT_GT(listener->compaction_reasons_.size(), 1);
|
|
|
|
|
|
|
|
for (auto compaction_reason : listener->compaction_reasons_) {
|
|
|
|
ASSERT_EQ(compaction_reason, CompactionReason::kLevelMaxLevelSize);
|
|
|
|
}
|
|
|
|
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
Close();
|
|
|
|
listener->compaction_reasons_.clear();
|
|
|
|
Reopen(options);
|
|
|
|
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(Put("key", "value"));
|
2016-04-26 03:18:35 +02:00
|
|
|
CompactRangeOptions cro;
|
2019-04-17 08:29:32 +02:00
|
|
|
cro.bottommost_level_compaction = BottommostLevelCompaction::kForceOptimized;
|
2016-04-26 03:18:35 +02:00
|
|
|
ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
|
2015-12-22 20:37:19 +01:00
|
|
|
ASSERT_GT(listener->compaction_reasons_.size(), 0);
|
|
|
|
for (auto compaction_reason : listener->compaction_reasons_) {
|
|
|
|
ASSERT_EQ(compaction_reason, CompactionReason::kManualCompaction);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, CompactionReasonUniversal) {
|
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2015-12-22 20:37:19 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.memtable_factory.reset(
|
|
|
|
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
|
|
|
|
|
|
|
|
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
|
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
|
|
|
|
options.compaction_style = kCompactionStyleUniversal;
|
|
|
|
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
options.level0_file_num_compaction_trigger = 8;
|
|
|
|
options.compaction_options_universal.max_size_amplification_percent = 100000;
|
|
|
|
options.compaction_options_universal.size_ratio = 100000;
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
listener->compaction_reasons_.clear();
|
|
|
|
|
|
|
|
// Write 8 files in L0
|
|
|
|
for (int i = 0; i < 8; i++) {
|
|
|
|
GenerateNewRandomFile(&rnd);
|
|
|
|
}
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-12-22 20:37:19 +01:00
|
|
|
|
|
|
|
ASSERT_GT(listener->compaction_reasons_.size(), 0);
|
|
|
|
for (auto compaction_reason : listener->compaction_reasons_) {
|
2018-01-26 20:01:54 +01:00
|
|
|
ASSERT_EQ(compaction_reason, CompactionReason::kUniversalSizeRatio);
|
2015-12-22 20:37:19 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
options.level0_file_num_compaction_trigger = 8;
|
|
|
|
options.compaction_options_universal.max_size_amplification_percent = 1;
|
|
|
|
options.compaction_options_universal.size_ratio = 100000;
|
|
|
|
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
listener->compaction_reasons_.clear();
|
|
|
|
|
|
|
|
// Write 8 files in L0
|
|
|
|
for (int i = 0; i < 8; i++) {
|
|
|
|
GenerateNewRandomFile(&rnd);
|
|
|
|
}
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-12-22 20:37:19 +01:00
|
|
|
|
|
|
|
ASSERT_GT(listener->compaction_reasons_.size(), 0);
|
|
|
|
for (auto compaction_reason : listener->compaction_reasons_) {
|
|
|
|
ASSERT_EQ(compaction_reason, CompactionReason::kUniversalSizeAmplification);
|
|
|
|
}
|
|
|
|
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
Close();
|
|
|
|
listener->compaction_reasons_.clear();
|
|
|
|
Reopen(options);
|
|
|
|
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
2015-12-22 20:37:19 +01:00
|
|
|
|
|
|
|
ASSERT_GT(listener->compaction_reasons_.size(), 0);
|
|
|
|
for (auto compaction_reason : listener->compaction_reasons_) {
|
|
|
|
ASSERT_EQ(compaction_reason, CompactionReason::kManualCompaction);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, CompactionReasonFIFO) {
|
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2015-12-22 20:37:19 +01:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.memtable_factory.reset(
|
|
|
|
new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
|
|
|
|
|
|
|
|
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
|
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
|
|
|
|
options.level0_file_num_compaction_trigger = 4;
|
|
|
|
options.compaction_style = kCompactionStyleFIFO;
|
|
|
|
options.compaction_options_fifo.max_table_files_size = 1;
|
|
|
|
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
Random rnd(301);
|
|
|
|
|
|
|
|
// Write 4 files in L0
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
|
|
GenerateNewRandomFile(&rnd);
|
|
|
|
}
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
2015-12-22 20:37:19 +01:00
|
|
|
|
|
|
|
ASSERT_GT(listener->compaction_reasons_.size(), 0);
|
|
|
|
for (auto compaction_reason : listener->compaction_reasons_) {
|
|
|
|
ASSERT_EQ(compaction_reason, CompactionReason::kFIFOMaxSize);
|
|
|
|
}
|
|
|
|
}
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
|
|
|
|
class TableFileCreationListener : public EventListener {
|
|
|
|
public:
|
|
|
|
class TestEnv : public EnvWrapper {
|
|
|
|
public:
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
explicit TestEnv(Env* t) : EnvWrapper(t) {}
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
|
|
|
|
void SetStatus(Status s) { status_ = s; }
|
|
|
|
|
|
|
|
Status NewWritableFile(const std::string& fname,
|
|
|
|
std::unique_ptr<WritableFile>* result,
|
2019-02-14 22:52:47 +01:00
|
|
|
const EnvOptions& options) override {
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
if (fname.size() > 4 && fname.substr(fname.size() - 4) == ".sst") {
|
|
|
|
if (!status_.ok()) {
|
|
|
|
return status_;
|
|
|
|
}
|
|
|
|
}
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
return target()->NewWritableFile(fname, result, options);
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
Status status_;
|
|
|
|
};
|
|
|
|
|
|
|
|
TableFileCreationListener() {
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
started_[i] = finished_[i] = failure_[i] = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int Index(TableFileCreationReason reason) {
|
|
|
|
int idx;
|
|
|
|
switch (reason) {
|
|
|
|
case TableFileCreationReason::kFlush:
|
|
|
|
idx = 0;
|
|
|
|
break;
|
|
|
|
case TableFileCreationReason::kCompaction:
|
|
|
|
idx = 1;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
idx = -1;
|
|
|
|
}
|
|
|
|
return idx;
|
|
|
|
}
|
|
|
|
|
|
|
|
void CheckAndResetCounters(int flush_started, int flush_finished,
|
|
|
|
int flush_failure, int compaction_started,
|
|
|
|
int compaction_finished, int compaction_failure) {
|
|
|
|
ASSERT_EQ(started_[0], flush_started);
|
|
|
|
ASSERT_EQ(finished_[0], flush_finished);
|
|
|
|
ASSERT_EQ(failure_[0], flush_failure);
|
|
|
|
ASSERT_EQ(started_[1], compaction_started);
|
|
|
|
ASSERT_EQ(finished_[1], compaction_finished);
|
|
|
|
ASSERT_EQ(failure_[1], compaction_failure);
|
|
|
|
for (int i = 0; i < 2; i++) {
|
|
|
|
started_[i] = finished_[i] = failure_[i] = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void OnTableFileCreationStarted(
|
|
|
|
const TableFileCreationBriefInfo& info) override {
|
|
|
|
int idx = Index(info.reason);
|
|
|
|
if (idx >= 0) {
|
|
|
|
started_[idx]++;
|
|
|
|
}
|
|
|
|
ASSERT_GT(info.db_name.size(), 0U);
|
|
|
|
ASSERT_GT(info.cf_name.size(), 0U);
|
|
|
|
ASSERT_GT(info.file_path.size(), 0U);
|
|
|
|
ASSERT_GT(info.job_id, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
void OnTableFileCreated(const TableFileCreationInfo& info) override {
|
|
|
|
int idx = Index(info.reason);
|
|
|
|
if (idx >= 0) {
|
|
|
|
finished_[idx]++;
|
|
|
|
}
|
|
|
|
ASSERT_GT(info.db_name.size(), 0U);
|
|
|
|
ASSERT_GT(info.cf_name.size(), 0U);
|
|
|
|
ASSERT_GT(info.file_path.size(), 0U);
|
|
|
|
ASSERT_GT(info.job_id, 0);
|
2020-08-25 19:44:39 +02:00
|
|
|
ASSERT_EQ(info.file_checksum, kUnknownFileChecksum);
|
|
|
|
ASSERT_EQ(info.file_checksum_func_name, kUnknownFileChecksumFuncName);
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
if (info.status.ok()) {
|
|
|
|
ASSERT_GT(info.table_properties.data_size, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.raw_key_size, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.raw_value_size, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.num_data_blocks, 0U);
|
|
|
|
ASSERT_GT(info.table_properties.num_entries, 0U);
|
|
|
|
} else {
|
|
|
|
if (idx >= 0) {
|
|
|
|
failure_[idx]++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int started_[2];
|
|
|
|
int finished_[2];
|
|
|
|
int failure_[2];
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, TableFileCreationListenersTest) {
|
|
|
|
auto listener = std::make_shared<TableFileCreationListener>();
|
|
|
|
Options options;
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
std::unique_ptr<TableFileCreationListener::TestEnv> test_env(
|
|
|
|
new TableFileCreationListener::TestEnv(CurrentOptions().env));
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.listeners.push_back(listener);
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.env = test_env.get();
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
DestroyAndReopen(options);
|
|
|
|
|
|
|
|
ASSERT_OK(Put("foo", "aaa"));
|
|
|
|
ASSERT_OK(Put("bar", "bbb"));
|
|
|
|
ASSERT_OK(Flush());
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
listener->CheckAndResetCounters(1, 1, 0, 0, 0, 0);
|
|
|
|
ASSERT_OK(Put("foo", "aaa1"));
|
|
|
|
ASSERT_OK(Put("bar", "bbb1"));
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
test_env->SetStatus(Status::NotSupported("not supported"));
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
ASSERT_NOK(Flush());
|
|
|
|
listener->CheckAndResetCounters(1, 1, 1, 0, 0, 0);
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
test_env->SetStatus(Status::OK());
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
|
|
|
|
Reopen(options);
|
|
|
|
ASSERT_OK(Put("foo", "aaa2"));
|
|
|
|
ASSERT_OK(Put("bar", "bbb2"));
|
|
|
|
ASSERT_OK(Flush());
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
listener->CheckAndResetCounters(1, 1, 0, 0, 0, 0);
|
|
|
|
|
|
|
|
const Slice kRangeStart = "a";
|
|
|
|
const Slice kRangeEnd = "z";
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(
|
|
|
|
dbfull()->CompactRange(CompactRangeOptions(), &kRangeStart, &kRangeEnd));
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
listener->CheckAndResetCounters(0, 0, 0, 1, 1, 0);
|
|
|
|
|
|
|
|
ASSERT_OK(Put("foo", "aaa3"));
|
|
|
|
ASSERT_OK(Put("bar", "bbb3"));
|
|
|
|
ASSERT_OK(Flush());
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
test_env->SetStatus(Status::NotSupported("not supported"));
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_NOK(
|
|
|
|
dbfull()->CompactRange(CompactRangeOptions(), &kRangeStart, &kRangeEnd));
|
|
|
|
ASSERT_NOK(dbfull()->TEST_WaitForCompact());
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
listener->CheckAndResetCounters(1, 1, 0, 1, 1, 1);
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
Close();
|
Added EventListener::OnTableFileCreationStarted() callback
Summary: Added EventListener::OnTableFileCreationStarted. EventListener::OnTableFileCreated will be called on failure case. User can check creation status via TableFileCreationInfo::status.
Test Plan: unit test.
Reviewers: dhruba, yhchiang, ott, sdong
Reviewed By: sdong
Subscribers: sdong, kradhakrishnan, IslamAbdelRahman, andrewkr, yhchiang, leveldb, ott, dhruba
Differential Revision: https://reviews.facebook.net/D56337
2016-04-29 20:35:00 +02:00
|
|
|
}
|
2016-06-02 20:57:31 +02:00
|
|
|
|
|
|
|
class MemTableSealedListener : public EventListener {
|
|
|
|
private:
|
|
|
|
SequenceNumber latest_seq_number_;
|
|
|
|
public:
|
|
|
|
MemTableSealedListener() {}
|
|
|
|
void OnMemTableSealed(const MemTableInfo& info) override {
|
|
|
|
latest_seq_number_ = info.first_seqno;
|
|
|
|
}
|
|
|
|
|
|
|
|
void OnFlushCompleted(DB* /*db*/,
|
|
|
|
const FlushJobInfo& flush_job_info) override {
|
|
|
|
ASSERT_LE(flush_job_info.smallest_seqno, latest_seq_number_);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, MemTableSealedListenerTest) {
|
|
|
|
auto listener = std::make_shared<MemTableSealedListener>();
|
|
|
|
Options options;
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.env = CurrentOptions().env;
|
2016-06-02 20:57:31 +02:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.listeners.push_back(listener);
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
|
|
|
|
for (unsigned int i = 0; i < 10; i++) {
|
|
|
|
std::string tag = std::to_string(i);
|
|
|
|
ASSERT_OK(Put("foo"+tag, "aaa"));
|
|
|
|
ASSERT_OK(Put("bar"+tag, "bbb"));
|
|
|
|
|
|
|
|
ASSERT_OK(Flush());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-09-22 20:56:18 +02:00
|
|
|
class ColumnFamilyHandleDeletionStartedListener : public EventListener {
|
|
|
|
private:
|
|
|
|
std::vector<std::string> cfs_;
|
|
|
|
int counter;
|
|
|
|
|
|
|
|
public:
|
|
|
|
explicit ColumnFamilyHandleDeletionStartedListener(
|
|
|
|
const std::vector<std::string>& cfs)
|
|
|
|
: cfs_(cfs), counter(0) {
|
|
|
|
cfs_.insert(cfs_.begin(), kDefaultColumnFamilyName);
|
|
|
|
}
|
|
|
|
void OnColumnFamilyHandleDeletionStarted(
|
|
|
|
ColumnFamilyHandle* handle) override {
|
|
|
|
ASSERT_EQ(cfs_[handle->GetID()], handle->GetName());
|
|
|
|
counter++;
|
|
|
|
}
|
|
|
|
int getCounter() { return counter; }
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, ColumnFamilyHandleDeletionStartedListenerTest) {
|
|
|
|
std::vector<std::string> cfs{"pikachu", "eevee", "Mewtwo"};
|
|
|
|
auto listener =
|
|
|
|
std::make_shared<ColumnFamilyHandleDeletionStartedListener>(cfs);
|
|
|
|
Options options;
|
2017-06-27 01:52:06 +02:00
|
|
|
options.env = CurrentOptions().env;
|
2016-09-22 20:56:18 +02:00
|
|
|
options.create_if_missing = true;
|
|
|
|
options.listeners.push_back(listener);
|
|
|
|
CreateAndReopenWithCF(cfs, options);
|
|
|
|
ASSERT_EQ(handles_.size(), 4);
|
|
|
|
delete handles_[3];
|
|
|
|
delete handles_[2];
|
|
|
|
delete handles_[1];
|
|
|
|
handles_.resize(1);
|
|
|
|
ASSERT_EQ(listener->getCounter(), 3);
|
|
|
|
}
|
|
|
|
|
2017-06-23 04:30:39 +02:00
|
|
|
class BackgroundErrorListener : public EventListener {
|
|
|
|
private:
|
|
|
|
SpecialEnv* env_;
|
|
|
|
int counter_;
|
|
|
|
|
|
|
|
public:
|
|
|
|
BackgroundErrorListener(SpecialEnv* env) : env_(env), counter_(0) {}
|
|
|
|
|
2018-03-05 22:08:17 +01:00
|
|
|
void OnBackgroundError(BackgroundErrorReason /*reason*/,
|
|
|
|
Status* bg_error) override {
|
2017-06-23 04:30:39 +02:00
|
|
|
if (counter_ == 0) {
|
|
|
|
// suppress the first error and disable write-dropping such that a retry
|
|
|
|
// can succeed.
|
|
|
|
*bg_error = Status::OK();
|
|
|
|
env_->drop_writes_.store(false, std::memory_order_release);
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
env_->SetMockSleep(false);
|
2017-06-23 04:30:39 +02:00
|
|
|
}
|
|
|
|
++counter_;
|
|
|
|
}
|
|
|
|
|
|
|
|
int counter() { return counter_; }
|
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, BackgroundErrorListenerFailedFlushTest) {
|
|
|
|
auto listener = std::make_shared<BackgroundErrorListener>(env_);
|
|
|
|
Options options;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.env = env_;
|
|
|
|
options.listeners.push_back(listener);
|
|
|
|
options.memtable_factory.reset(new SpecialSkipListFactory(1));
|
|
|
|
options.paranoid_checks = true;
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
|
|
|
|
// the usual TEST_WaitForFlushMemTable() doesn't work for failed flushes, so
|
|
|
|
// forge a custom one for the failed flush case.
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
|
2017-06-23 04:30:39 +02:00
|
|
|
{{"DBImpl::BGWorkFlush:done",
|
|
|
|
"EventListenerTest:BackgroundErrorListenerFailedFlushTest:1"}});
|
2020-02-20 21:07:53 +01:00
|
|
|
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
|
2017-06-23 04:30:39 +02:00
|
|
|
|
|
|
|
env_->drop_writes_.store(true, std::memory_order_release);
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
env_->SetMockSleep();
|
2017-06-23 04:30:39 +02:00
|
|
|
|
|
|
|
ASSERT_OK(Put("key0", "val"));
|
|
|
|
ASSERT_OK(Put("key1", "val"));
|
|
|
|
TEST_SYNC_POINT("EventListenerTest:BackgroundErrorListenerFailedFlushTest:1");
|
|
|
|
ASSERT_EQ(1, listener->counter());
|
|
|
|
ASSERT_OK(Put("key2", "val"));
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
|
|
ASSERT_EQ(1, NumTableFilesAtLevel(0));
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, BackgroundErrorListenerFailedCompactionTest) {
|
|
|
|
auto listener = std::make_shared<BackgroundErrorListener>(env_);
|
|
|
|
Options options;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
options.disable_auto_compactions = true;
|
|
|
|
options.env = env_;
|
|
|
|
options.level0_file_num_compaction_trigger = 2;
|
|
|
|
options.listeners.push_back(listener);
|
|
|
|
options.memtable_factory.reset(new SpecialSkipListFactory(2));
|
|
|
|
options.paranoid_checks = true;
|
|
|
|
DestroyAndReopen(options);
|
|
|
|
|
|
|
|
// third iteration triggers the second memtable's flush
|
|
|
|
for (int i = 0; i < 3; ++i) {
|
|
|
|
ASSERT_OK(Put("key0", "val"));
|
|
|
|
if (i > 0) {
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
|
|
}
|
|
|
|
ASSERT_OK(Put("key1", "val"));
|
|
|
|
}
|
|
|
|
ASSERT_EQ(2, NumTableFilesAtLevel(0));
|
|
|
|
|
|
|
|
env_->drop_writes_.store(true, std::memory_order_release);
|
Fix+clean up handling of mock sleeps (#7101)
Summary:
We have a number of tests hanging on MacOS and windows due to
mishandling of code for mock sleeps. In addition, the code was in
terrible shape because the same variable (addon_time_) would sometimes
refer to microseconds and sometimes to seconds. One test even assumed it
was nanoseconds but was written to pass anyway.
This has been cleaned up so that DB tests generally use a SpecialEnv
function to mock sleep, for either some number of microseconds or seconds
depending on the function called. But to call one of these, the test must first
call SetMockSleep (precondition enforced with assertion), which also turns
sleeps in RocksDB into mock sleeps. To also removes accounting for actual
clock time, call SetTimeElapseOnlySleepOnReopen, which implies
SetMockSleep (on DB re-open). This latter setting only works by applying
on DB re-open, otherwise havoc can ensue if Env goes back in time with
DB open.
More specifics:
Removed some unused test classes, and updated comments on the general
problem.
Fixed DBSSTTest.GetTotalSstFilesSize using a sync point callback instead
of mock time. For this we have the only modification to production code,
inserting a sync point callback in flush_job.cc, which is not a change to
production behavior.
Removed unnecessary resetting of mock times to 0 in many tests. RocksDB
deals in relative time. Any behaviors relying on absolute date/time are likely
a bug. (The above test DBSSTTest.GetTotalSstFilesSize was the only one
clearly injecting a specific absolute time for actual testing convenience.) Just
in case I misunderstood some test, I put this note in each replacement:
// NOTE: Presumed unnecessary and removed: resetting mock time in env
Strengthened some tests like MergeTestTime, MergeCompactionTimeTest, and
FilterCompactionTimeTest in db_test.cc
stats_history_test and blob_db_test are each their own beast, rather deeply
dependent on MockTimeEnv. Each gets its own variant of a work-around for
TimedWait in a mock time environment. (Reduces redundancy and
inconsistency in stats_history_test.)
Intended follow-up:
Remove TimedWait from the public API of InstrumentedCondVar, and only
make that accessible through Env by passing in an InstrumentedCondVar and
a deadline. Then the Env implementations mocking time can fix this problem
without using sync points. (Test infrastructure using sync points interferes
with individual tests' control over sync points.)
With that change, we can simplify/consolidate the scattered work-arounds.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7101
Test Plan: make check on Linux and MacOS
Reviewed By: zhichao-cao
Differential Revision: D23032815
Pulled By: pdillinger
fbshipit-source-id: 7f33967ada8b83011fb54e8279365c008bd6610b
2020-08-11 21:39:49 +02:00
|
|
|
env_->SetMockSleep();
|
2017-06-23 04:30:39 +02:00
|
|
|
ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}}));
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
|
|
|
ASSERT_EQ(1, listener->counter());
|
|
|
|
|
|
|
|
// trigger flush so compaction is triggered again; this time it succeeds
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 22:36:19 +02:00
|
|
|
// The previous failed compaction may get retried automatically, so we may
|
|
|
|
// be left with 0 or 1 files in level 1, depending on when the retry gets
|
|
|
|
// scheduled
|
2017-06-23 04:30:39 +02:00
|
|
|
ASSERT_OK(Put("key0", "val"));
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForCompact());
|
Auto recovery from out of space errors (#4164)
Summary:
This commit implements automatic recovery from a Status::NoSpace() error
during background operations such as write callback, flush and
compaction. The broad design is as follows -
1. Compaction errors are treated as soft errors and don't put the
database in read-only mode. A compaction is delayed until enough free
disk space is available to accomodate the compaction outputs, which is
estimated based on the input size. This means that users can continue to
write, and we rely on the WriteController to delay or stop writes if the
compaction debt becomes too high due to persistent low disk space
condition
2. Errors during write callback and flush are treated as hard errors,
i.e the database is put in read-only mode and goes back to read-write
only fater certain recovery actions are taken.
3. Both types of recovery rely on the SstFileManagerImpl to poll for
sufficient disk space. We assume that there is a 1-1 mapping between an
SFM and the underlying OS storage container. For cases where multiple
DBs are hosted on a single storage container, the user is expected to
allocate a single SFM instance and use the same one for all the DBs. If
no SFM is specified by the user, DBImpl::Open() will allocate one, but
this will be one per DB and each DB will recover independently. The
recovery implemented by SFM is as follows -
a) On the first occurance of an out of space error during compaction,
subsequent
compactions will be delayed until the disk free space check indicates
enough available space. The required space is computed as the sum of
input sizes.
b) The free space check requirement will be removed once the amount of
free space is greater than the size reserved by in progress
compactions when the first error occured
c) If the out of space error is a hard error, a background thread in
SFM will poll for sufficient headroom before triggering the recovery
of the database and putting it in write-only mode. The headroom is
calculated as the sum of the write_buffer_size of all the DB instances
associated with the SFM
4. EventListener callbacks will be called at the start and completion of
automatic recovery. Users can disable the auto recov ery in the start
callback, and later initiate it manually by calling DB::Resume()
Todo:
1. More extensive testing
2. Add disk full condition to db_stress (follow-on PR)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4164
Differential Revision: D9846378
Pulled By: anand1976
fbshipit-source-id: 80ea875dbd7f00205e19c82215ff6e37da10da4a
2018-09-15 22:36:19 +02:00
|
|
|
ASSERT_LE(1, NumTableFilesAtLevel(0));
|
2017-06-23 04:30:39 +02:00
|
|
|
}
|
|
|
|
|
2018-10-13 03:34:03 +02:00
|
|
|
class TestFileOperationListener : public EventListener {
|
|
|
|
public:
|
|
|
|
TestFileOperationListener() {
|
|
|
|
file_reads_.store(0);
|
|
|
|
file_reads_success_.store(0);
|
|
|
|
file_writes_.store(0);
|
|
|
|
file_writes_success_.store(0);
|
2020-07-08 03:19:32 +02:00
|
|
|
file_flushes_.store(0);
|
|
|
|
file_flushes_success_.store(0);
|
|
|
|
file_closes_.store(0);
|
|
|
|
file_closes_success_.store(0);
|
|
|
|
file_syncs_.store(0);
|
|
|
|
file_syncs_success_.store(0);
|
|
|
|
file_truncates_.store(0);
|
|
|
|
file_truncates_success_.store(0);
|
2018-10-13 03:34:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void OnFileReadFinish(const FileOperationInfo& info) override {
|
|
|
|
++file_reads_;
|
|
|
|
if (info.status.ok()) {
|
|
|
|
++file_reads_success_;
|
|
|
|
}
|
2019-01-16 18:48:01 +01:00
|
|
|
ReportDuration(info);
|
2018-10-13 03:34:03 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
void OnFileWriteFinish(const FileOperationInfo& info) override {
|
|
|
|
++file_writes_;
|
|
|
|
if (info.status.ok()) {
|
|
|
|
++file_writes_success_;
|
|
|
|
}
|
2019-01-16 18:48:01 +01:00
|
|
|
ReportDuration(info);
|
2018-10-13 03:34:03 +02:00
|
|
|
}
|
|
|
|
|
2020-07-08 03:19:32 +02:00
|
|
|
void OnFileFlushFinish(const FileOperationInfo& info) override {
|
|
|
|
++file_flushes_;
|
|
|
|
if (info.status.ok()) {
|
|
|
|
++file_flushes_success_;
|
|
|
|
}
|
|
|
|
ReportDuration(info);
|
|
|
|
}
|
|
|
|
|
|
|
|
void OnFileCloseFinish(const FileOperationInfo& info) override {
|
|
|
|
++file_closes_;
|
|
|
|
if (info.status.ok()) {
|
|
|
|
++file_closes_success_;
|
|
|
|
}
|
|
|
|
ReportDuration(info);
|
|
|
|
}
|
|
|
|
|
|
|
|
void OnFileSyncFinish(const FileOperationInfo& info) override {
|
|
|
|
++file_syncs_;
|
|
|
|
if (info.status.ok()) {
|
|
|
|
++file_syncs_success_;
|
|
|
|
}
|
|
|
|
ReportDuration(info);
|
|
|
|
}
|
|
|
|
|
|
|
|
void OnFileTruncateFinish(const FileOperationInfo& info) override {
|
|
|
|
++file_truncates_;
|
|
|
|
if (info.status.ok()) {
|
|
|
|
++file_truncates_success_;
|
|
|
|
}
|
|
|
|
ReportDuration(info);
|
|
|
|
}
|
|
|
|
|
2018-10-13 03:34:03 +02:00
|
|
|
bool ShouldBeNotifiedOnFileIO() override { return true; }
|
|
|
|
|
|
|
|
std::atomic<size_t> file_reads_;
|
|
|
|
std::atomic<size_t> file_reads_success_;
|
|
|
|
std::atomic<size_t> file_writes_;
|
|
|
|
std::atomic<size_t> file_writes_success_;
|
2020-07-08 03:19:32 +02:00
|
|
|
std::atomic<size_t> file_flushes_;
|
|
|
|
std::atomic<size_t> file_flushes_success_;
|
|
|
|
std::atomic<size_t> file_closes_;
|
|
|
|
std::atomic<size_t> file_closes_success_;
|
|
|
|
std::atomic<size_t> file_syncs_;
|
|
|
|
std::atomic<size_t> file_syncs_success_;
|
|
|
|
std::atomic<size_t> file_truncates_;
|
|
|
|
std::atomic<size_t> file_truncates_success_;
|
2019-01-16 18:48:01 +01:00
|
|
|
|
|
|
|
private:
|
|
|
|
void ReportDuration(const FileOperationInfo& info) const {
|
2020-07-22 17:53:21 +02:00
|
|
|
ASSERT_GT(info.duration.count(), 0);
|
2019-01-16 18:48:01 +01:00
|
|
|
}
|
2018-10-13 03:34:03 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
TEST_F(EventListenerTest, OnFileOperationTest) {
|
|
|
|
Options options;
|
|
|
|
options.env = CurrentOptions().env;
|
|
|
|
options.create_if_missing = true;
|
|
|
|
|
|
|
|
TestFileOperationListener* listener = new TestFileOperationListener();
|
|
|
|
options.listeners.emplace_back(listener);
|
|
|
|
|
Fix many tests to run with MEM_ENV and ENCRYPTED_ENV; Introduce a MemoryFileSystem class (#7566)
Summary:
This PR does a few things:
1. The MockFileSystem class was split out from the MockEnv. This change would theoretically allow a MockFileSystem to be used by other Environments as well (if we created a means of constructing one). The MockFileSystem implements a FileSystem in its entirety and does not rely on any Wrapper implementation.
2. Make the RocksDB test suite work when MOCK_ENV=1 and ENCRYPTED_ENV=1 are set. To accomplish this, a few things were needed:
- The tests that tried to use the "wrong" environment (Env::Default() instead of env_) were updated
- The MockFileSystem was changed to support the features it was missing or mishandled (such as recursively deleting files in a directory or supporting renaming of a directory).
3. Updated the test framework to have a ROCKSDB_GTEST_SKIP macro. This can be used to flag tests that are skipped. Currently, this defaults to doing nothing (marks the test as SUCCESS) but will mark the tests as SKIPPED when RocksDB is upgraded to a version of gtest that supports this (gtest-1.10).
I have run a full "make check" with MEM_ENV, ENCRYPTED_ENV, both, and neither under both MacOS and RedHat. A few tests were disabled/skipped for the MEM/ENCRYPTED cases. The error_handler_fs_test fails/hangs for MEM_ENV (presumably a timing problem) and I will introduce another PR/issue to track that problem. (I will also push a change to disable those tests soon). There is one more test in DBTest2 that also fails which I need to investigate or skip before this PR is merged.
Theoretically, this PR should also allow the test suite to run against an Env loaded from the registry, though I do not have one to try it with currently.
Finally, once this is accepted, it would be nice if there was a CircleCI job to run these tests on a checkin so this effort does not become stale. I do not know how to do that, so if someone could write that job, it would be appreciated :)
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7566
Reviewed By: zhichao-cao
Differential Revision: D24408980
Pulled By: jay-zhuang
fbshipit-source-id: 911b1554a4d0da06fd51feca0c090a4abdcb4a5f
2020-10-27 18:31:34 +01:00
|
|
|
options.use_direct_io_for_flush_and_compaction = false;
|
2020-07-08 03:19:32 +02:00
|
|
|
Status s = TryReopen(options);
|
|
|
|
if (s.IsInvalidArgument()) {
|
|
|
|
options.use_direct_io_for_flush_and_compaction = false;
|
|
|
|
} else {
|
|
|
|
ASSERT_OK(s);
|
|
|
|
}
|
2018-10-13 03:34:03 +02:00
|
|
|
DestroyAndReopen(options);
|
|
|
|
ASSERT_OK(Put("foo", "aaa"));
|
2020-12-23 08:44:44 +01:00
|
|
|
ASSERT_OK(dbfull()->Flush(FlushOptions()));
|
|
|
|
ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
|
2018-10-13 03:34:03 +02:00
|
|
|
ASSERT_GE(listener->file_writes_.load(),
|
|
|
|
listener->file_writes_success_.load());
|
|
|
|
ASSERT_GT(listener->file_writes_.load(), 0);
|
2020-07-08 03:19:32 +02:00
|
|
|
ASSERT_GE(listener->file_flushes_.load(),
|
|
|
|
listener->file_flushes_success_.load());
|
|
|
|
ASSERT_GT(listener->file_flushes_.load(), 0);
|
2018-10-13 03:34:03 +02:00
|
|
|
Close();
|
|
|
|
|
|
|
|
Reopen(options);
|
|
|
|
ASSERT_GE(listener->file_reads_.load(), listener->file_reads_success_.load());
|
|
|
|
ASSERT_GT(listener->file_reads_.load(), 0);
|
2020-07-08 03:19:32 +02:00
|
|
|
ASSERT_GE(listener->file_closes_.load(),
|
|
|
|
listener->file_closes_success_.load());
|
|
|
|
ASSERT_GT(listener->file_closes_.load(), 0);
|
|
|
|
ASSERT_GE(listener->file_syncs_.load(), listener->file_syncs_success_.load());
|
|
|
|
ASSERT_GT(listener->file_syncs_.load(), 0);
|
|
|
|
if (true == options.use_direct_io_for_flush_and_compaction) {
|
|
|
|
ASSERT_GE(listener->file_truncates_.load(),
|
|
|
|
listener->file_truncates_success_.load());
|
|
|
|
ASSERT_GT(listener->file_truncates_.load(), 0);
|
|
|
|
}
|
2018-10-13 03:34:03 +02:00
|
|
|
}
|
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
2015-03-17 22:08:00 +01:00
|
|
|
::testing::InitGoogleTest(&argc, argv);
|
|
|
|
return RUN_ALL_TESTS();
|
CompactFiles, EventListener and GetDatabaseMetaData
Summary:
This diff adds three sets of APIs to RocksDB.
= GetColumnFamilyMetaData =
* This APIs allow users to obtain the current state of a RocksDB instance on one column family.
* See GetColumnFamilyMetaData in include/rocksdb/db.h
= EventListener =
* A virtual class that allows users to implement a set of
call-back functions which will be called when specific
events of a RocksDB instance happens.
* To register EventListener, simply insert an EventListener to ColumnFamilyOptions::listeners
= CompactFiles =
* CompactFiles API inputs a set of file numbers and an output level, and RocksDB
will try to compact those files into the specified level.
= Example =
* Example code can be found in example/compact_files_example.cc, which implements
a simple external compactor using EventListener, GetColumnFamilyMetaData, and
CompactFiles API.
Test Plan:
listener_test
compactor_test
example/compact_files_example
export ROCKSDB_TESTS=CompactFiles
db_test
export ROCKSDB_TESTS=MetaData
db_test
Reviewers: ljin, igor, rven, sdong
Reviewed By: sdong
Subscribers: MarkCallaghan, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D24705
2014-11-07 23:45:18 +01:00
|
|
|
}
|