[column families] Support to create and drop column families

Summary:
This diff provides basic implementations of CreateColumnFamily(), DropColumnFamily() and ListColumnFamilies(). It builds on top of https://reviews.facebook.net/D14733

It also includes a bug fix for DBImplReadOnly, where Get implementation would be redirected to DBImpl instead of DBImplReadOnly.

Test Plan: Added unit test

Reviewers: dhruba, haobo, kailiu

CC: leveldb

Differential Revision: https://reviews.facebook.net/D15021
This commit is contained in:
Igor Canadi 2014-01-02 09:08:12 -08:00
parent 7535443083
commit ef6ad1708d
9 changed files with 180 additions and 31 deletions

View File

@ -51,6 +51,7 @@ VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
TESTS = \ TESTS = \
db_test \ db_test \
autovector_test \ autovector_test \
column_family_test \
table_properties_collector_test \ table_properties_collector_test \
arena_test \ arena_test \
auto_roll_logger_test \ auto_roll_logger_test \
@ -230,6 +231,9 @@ arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
autovector_test: util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS) autovector_test: util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) $(CXX) util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
column_family_test: db/column_family_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/column_family_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
table_properties_collector_test: db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) table_properties_collector_test: db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) $(CXX) db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

73
db/column_family_test.cc Normal file
View File

@ -0,0 +1,73 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "db/db_impl.h"
#include "rocksdb/db.h"
#include "util/testharness.h"
#include <algorithm>
#include <vector>
#include <string>
namespace rocksdb {
using namespace std;
class ColumnFamilyTest {
public:
ColumnFamilyTest() {
dbname_ = test::TmpDir() + "/column_family_test";
db_options_.create_if_missing = true;
options_.create_if_missing = true;
DestroyDB(dbname_, options_);
}
void Close() {
delete db_;
db_ = nullptr;
}
void Open() {
ASSERT_OK(DB::Open(options_, dbname_, &db_));
}
Options options_;
ColumnFamilyOptions column_family_options_;
DBOptions db_options_;
string dbname_;
DB* db_;
};
TEST(ColumnFamilyTest, AddDrop) {
Open();
ColumnFamilyHandle handles[4];
ASSERT_OK(db_->CreateColumnFamily(column_family_options_, Slice("one"),
&handles[0]));
ASSERT_OK(db_->CreateColumnFamily(column_family_options_, Slice("two"),
&handles[1]));
ASSERT_OK(db_->CreateColumnFamily(column_family_options_, Slice("three"),
&handles[2]));
ASSERT_OK(db_->DropColumnFamily(handles[1]));
ASSERT_OK(db_->CreateColumnFamily(column_family_options_, Slice("four"),
&handles[3]));
Close();
Open(); // this will roll the manifest, column families should stay consistent
Close();
vector<string> families;
DB::ListColumnFamilies(db_options_, dbname_, &families);
sort(families.begin(), families.end());
ASSERT_TRUE(families == vector<string>({"four", "one", "three"}));
}
} // namespace rocksdb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
}

View File

@ -2856,6 +2856,26 @@ std::vector<Status> DBImpl::MultiGet(
return statList; return statList;
} }
Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& options,
const Slice& column_family,
ColumnFamilyHandle* handle) {
VersionEdit edit(0);
edit.AddColumnFamily(column_family.ToString());
MutexLock l(&mutex_);
++versions_->max_column_family_;
handle->id = versions_->max_column_family_;
edit.SetColumnFamily(handle->id);
return versions_->LogAndApply(&edit, &mutex_);
}
Status DBImpl::DropColumnFamily(const ColumnFamilyHandle& column_family) {
VersionEdit edit(0);
edit.DropColumnFamily();
edit.SetColumnFamily(column_family.id);
MutexLock l(&mutex_);
return versions_->LogAndApply(&edit, &mutex_);
}
bool DBImpl::KeyMayExist(const ReadOptions& options, bool DBImpl::KeyMayExist(const ReadOptions& options,
const ColumnFamilyHandle& column_family, const ColumnFamilyHandle& column_family,
const Slice& key, std::string* value, const Slice& key, std::string* value,
@ -3776,14 +3796,14 @@ Status DB::Merge(const WriteOptions& opt,
return Write(opt, &batch); return Write(opt, &batch);
} }
Status DB::OpenColumnFamily(const ColumnFamilyOptions& options, // Default implementation -- returns not supported status
const Slice& column_family, Status DB::CreateColumnFamily(const ColumnFamilyOptions& options,
ColumnFamilyHandle* handle) { const Slice& column_family,
return Status::NotSupported("working on it"); ColumnFamilyHandle* handle) {
return Status::NotSupported("");
} }
Status DB::DropColumnFamily(const ColumnFamilyHandle& column_family) { Status DB::DropColumnFamily(const ColumnFamilyHandle& column_family) {
return Status::NotSupported("working on it"); return Status::NotSupported("");
} }
DB::~DB() { } DB::~DB() { }
@ -3866,10 +3886,25 @@ Status DB::OpenWithColumnFamilies(
return Status::NotSupported("Working on it"); return Status::NotSupported("Working on it");
} }
Status DB::ListColumnFamilies( Status DB::ListColumnFamilies(const DBOptions& db_options,
const DBOptions& db_options, const std::string& name, const std::string& name,
const std::vector<std::string>* column_families) { std::vector<std::string>* column_families) {
// TODO Options options(db_options, ColumnFamilyOptions());
InternalKeyComparator* icmp = new InternalKeyComparator(options.comparator);
TableCache* table_cache = new TableCache(name, &options, EnvOptions(options),
db_options.max_open_files - 10);
VersionSet* version_set =
new VersionSet(name, &options, EnvOptions(options), table_cache, icmp);
version_set->Recover();
column_families->clear();
for (auto cf : version_set->column_families_) {
column_families->push_back(cf.first);
}
delete version_set;
delete table_cache;
delete icmp;
return Status::NotSupported("Working on it"); return Status::NotSupported("Working on it");
} }

View File

@ -61,6 +61,11 @@ class DBImpl : public DB {
const std::vector<ColumnFamilyHandle>& column_family, const std::vector<ColumnFamilyHandle>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values); const std::vector<Slice>& keys, std::vector<std::string>* values);
virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
const Slice& column_family,
ColumnFamilyHandle* handle);
virtual Status DropColumnFamily(const ColumnFamilyHandle& column_family);
// Returns false if key doesn't exist in the database and true if it may. // Returns false if key doesn't exist in the database and true if it may.
// If value_found is not passed in as null, then return the value if found in // If value_found is not passed in as null, then return the value if found in
// memory. On return, if value was found, then value_found will be set to true // memory. On return, if value was found, then value_found will be set to true

View File

@ -52,8 +52,9 @@ DBImplReadOnly::~DBImplReadOnly() {
} }
// Implementations of the DB interface // Implementations of the DB interface
Status DBImplReadOnly::Get(const ReadOptions& options, const Slice& key, Status DBImplReadOnly::Get(const ReadOptions& options,
std::string* value) { const ColumnFamilyHandle& column_family,
const Slice& key, std::string* value) {
Status s; Status s;
MemTable* mem = GetMemTable(); MemTable* mem = GetMemTable();
Version* current = versions_->current(); Version* current = versions_->current();

View File

@ -28,9 +28,9 @@ public:
virtual ~DBImplReadOnly(); virtual ~DBImplReadOnly();
// Implementations of the DB interface // Implementations of the DB interface
using DBImpl::Get; using DB::Get;
virtual Status Get(const ReadOptions& options, virtual Status Get(const ReadOptions& options,
const Slice& key, const ColumnFamilyHandle& column_family, const Slice& key,
std::string* value); std::string* value);
// TODO: Implement ReadOnly MultiGet? // TODO: Implement ReadOnly MultiGet?

View File

@ -1466,6 +1466,23 @@ Status VersionSet::Recover() {
builder.Apply(&edit); builder.Apply(&edit);
} }
if (edit.is_column_family_add_) {
assert(column_families_.find(edit.column_family_name_) ==
column_families_.end());
column_families_.insert(
{edit.column_family_name_, edit.column_family_});
column_family_data_.insert(
{edit.column_family_, ColumnFamilyData(edit.column_family_name_)});
max_column_family_ = std::max(max_column_family_, edit.column_family_);
}
if (edit.is_column_family_drop_) {
auto cf = column_family_data_.find(edit.column_family_);
assert(cf != column_family_data_.end());
column_families_.erase(cf->second.name);
column_family_data_.erase(cf);
}
if (edit.has_log_number_) { if (edit.has_log_number_) {
log_number = edit.log_number_; log_number = edit.log_number_;
have_log_number = true; have_log_number = true;
@ -1827,6 +1844,19 @@ void VersionSet::UpdateFilesBySize(Version* v) {
Status VersionSet::WriteSnapshot(log::Writer* log) { Status VersionSet::WriteSnapshot(log::Writer* log) {
// TODO: Break up into multiple records to reduce memory usage on recovery? // TODO: Break up into multiple records to reduce memory usage on recovery?
// Save column families
for (auto cf : column_families_) {
VersionEdit edit(0);
edit.AddColumnFamily(cf.first);
edit.SetColumnFamily(cf.second);
std::string record;
edit.EncodeTo(&record);
Status s = log->AddRecord(record);
if (!s.ok()) {
return s;
}
}
// Save metadata // Save metadata
VersionEdit edit(NumberLevels()); VersionEdit edit(NumberLevels());
edit.SetComparatorName(icmp_.user_comparator()->Name()); edit.SetComparatorName(icmp_.user_comparator()->Name());

View File

@ -436,6 +436,15 @@ class VersionSet {
void GetObsoleteFiles(std::vector<FileMetaData*>* files); void GetObsoleteFiles(std::vector<FileMetaData*>* files);
// column family metadata
struct ColumnFamilyData {
std::string name;
explicit ColumnFamilyData(const std::string& name) : name(name) {}
};
std::unordered_map<std::string, uint32_t> column_families_;
std::unordered_map<uint32_t, ColumnFamilyData> column_family_data_;
uint32_t max_column_family_;
private: private:
class Builder; class Builder;
struct ManifestWriter; struct ManifestWriter;
@ -505,10 +514,6 @@ class VersionSet {
// generates a increasing version number for every new version // generates a increasing version number for every new version
uint64_t current_version_number_; uint64_t current_version_number_;
// column family metadata
std::unordered_map<std::string, ColumnFamilyHandle> column_families_;
uint32_t max_column_family_id_;
// Queue of writers to the manifest file // Queue of writers to the manifest file
std::deque<ManifestWriter*> manifest_writers_; std::deque<ManifestWriter*> manifest_writers_;

View File

@ -107,28 +107,24 @@ class DB {
// and return the list of all column families in that DB // and return the list of all column families in that DB
// through column_families argument. The ordering of // through column_families argument. The ordering of
// column families in column_families is unspecified. // column families in column_families is unspecified.
static Status ListColumnFamilies( static Status ListColumnFamilies(const DBOptions& db_options,
const DBOptions& db_options, const std::string& name, const std::string& name,
const std::vector<std::string>* column_families); std::vector<std::string>* column_families);
DB() { } DB() { }
virtual ~DB(); virtual ~DB();
// Open a column_family and return the handle of column family // Create a column_family and return the handle of column family
// through the argument handle // through the argument handle.
// If the column family already exists in the Database, virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
// it will open it and make it available for the client to query. const Slice& column_family,
// If the column family does not exist, the function will create ColumnFamilyHandle* handle);
// and persist it.
Status OpenColumnFamily(const ColumnFamilyOptions& options,
const Slice& column_family,
ColumnFamilyHandle* handle);
// Drop a column family specified by column_family handle. // Drop a column family specified by column_family handle.
// All data related to the column family will be deleted before // All data related to the column family will be deleted before
// the function returns. // the function returns.
// Calls referring to the dropped column family will fail. // Calls referring to the dropped column family will fail.
Status DropColumnFamily(const ColumnFamilyHandle& column_family); virtual Status DropColumnFamily(const ColumnFamilyHandle& column_family);
// Set the database entry for "key" to "value". // Set the database entry for "key" to "value".
// Returns OK on success, and a non-OK status on error. // Returns OK on success, and a non-OK status on error.