From b947fdc89dd01933b7f7abed85b12e8fd5d8c184 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Wed, 9 Apr 2014 09:56:17 -0700 Subject: [PATCH] Column family support for DB::OpenForReadOnly() Summary: When opening DB in read-only mode, client can choose to only specify a subset of column families ("default" column family can't be omitted, though) Test Plan: added a unit test in column_family_test Reviewers: haobo, sdong, ljin, dhruba Reviewed By: haobo CC: leveldb Differential Revision: https://reviews.facebook.net/D17565 --- db/column_family_test.cc | 45 ++++++++++++++++++++++++++++++++++++++++ db/db_impl.cc | 8 +++---- db/db_impl_readonly.cc | 39 +++++++++++++++++++++++++++++++++- db/db_test.cc | 3 +-- db/version_set.cc | 26 ++++++++++++++--------- db/version_set.h | 5 ++++- include/rocksdb/db.h | 28 ++++++++++++++++++------- tools/db_stress.cc | 4 ++-- util/ldb_cmd.cc | 2 +- 9 files changed, 131 insertions(+), 29 deletions(-) diff --git a/db/column_family_test.cc b/db/column_family_test.cc index 16e98629f..57acb1b5c 100644 --- a/db/column_family_test.cc +++ b/db/column_family_test.cc @@ -60,6 +60,25 @@ class ColumnFamilyTest { return DB::Open(db_options_, dbname_, column_families, &handles_, &db_); } + Status OpenReadOnly(std::vector cf, + std::vector options = {}) { + std::vector column_families; + names_.clear(); + for (size_t i = 0; i < cf.size(); ++i) { + column_families.push_back(ColumnFamilyDescriptor( + cf[i], options.size() == 0 ? column_family_options_ : options[i])); + names_.push_back(cf[i]); + } + return DB::OpenForReadOnly(db_options_, dbname_, column_families, &handles_, + &db_); + } + + void AssertOpenReadOnly(std::vector cf, + std::vector options = {}) { + ASSERT_OK(OpenReadOnly(cf, options)); + } + + void Open(std::vector cf, std::vector options = {}) { ASSERT_OK(TryOpen(cf, options)); @@ -850,6 +869,32 @@ TEST(ColumnFamilyTest, NewIteratorsTest) { } } +TEST(ColumnFamilyTest, ReadOnlyDBTest) { + Open(); + CreateColumnFamiliesAndReopen({"one", "two", "three", "four"}); + ASSERT_OK(Put(1, "foo", "bla")); + ASSERT_OK(Put(2, "foo", "blabla")); + ASSERT_OK(Put(3, "foo", "blablabla")); + ASSERT_OK(Put(4, "foo", "blablablabla")); + + DropColumnFamilies({2}); + Close(); + // open only a subset of column families + AssertOpenReadOnly({"default", "one", "four"}); + ASSERT_EQ("NOT_FOUND", Get(0, "foo")); + ASSERT_EQ("bla", Get(1, "foo")); + ASSERT_EQ("blablablabla", Get(2, "foo")); + + Close(); + // can't open dropped column family + Status s = OpenReadOnly({"default", "one", "two"}); + ASSERT_TRUE(!s.ok()); + + // Can't open without specifying default column family + s = OpenReadOnly({"one", "four"}); + ASSERT_TRUE(!s.ok()); +} + } // namespace rocksdb int main(int argc, char** argv) { diff --git a/db/db_impl.cc b/db/db_impl.cc index c0d8440dd..1061c4880 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -70,7 +70,7 @@ namespace rocksdb { -const std::string default_column_family_name("default"); +const std::string kDefaultColumnFamilyName("default"); void DumpLeveldbBuildVersion(Logger * log); @@ -949,7 +949,7 @@ Status DBImpl::Recover( } } - Status s = versions_->Recover(column_families); + Status s = versions_->Recover(column_families, read_only); if (options_.paranoid_checks && s.ok()) { s = CheckConsistency(); } @@ -4498,7 +4498,7 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) { ColumnFamilyOptions cf_options(options); std::vector column_families; column_families.push_back( - ColumnFamilyDescriptor(default_column_family_name, cf_options)); + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); std::vector handles; Status s = DB::Open(db_options, dbname, column_families, &handles, dbptr); if (s.ok()) { @@ -4568,8 +4568,8 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname, if (s.ok()) { for (auto cfd : *impl->versions_->GetColumnFamilySet()) { delete cfd->InstallSuperVersion(new SuperVersion(), &impl->mutex_); - impl->alive_log_files_.push_back(impl->logfile_number_); } + impl->alive_log_files_.push_back(impl->logfile_number_); impl->DeleteObsoleteFiles(); impl->MaybeScheduleFlushOrCompaction(); impl->MaybeScheduleLogDBDeployStats(); diff --git a/db/db_impl_readonly.cc b/db/db_impl_readonly.cc index 6d519a07c..d016931e8 100644 --- a/db/db_impl_readonly.cc +++ b/db/db_impl_readonly.cc @@ -94,12 +94,44 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname, ColumnFamilyOptions cf_options(options); std::vector column_families; column_families.push_back( - ColumnFamilyDescriptor(default_column_family_name, cf_options)); + ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); + std::vector handles; + + Status s = + DB::OpenForReadOnly(db_options, dbname, column_families, &handles, dbptr); + if (s.ok()) { + assert(handles.size() == 1); + // i can delete the handle since DBImpl is always holding a + // reference to default column family + delete handles[0]; + } + return s; +} + +Status DB::OpenForReadOnly( + const DBOptions& db_options, const std::string& dbname, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + bool error_if_log_file_exist) { + *dbptr = nullptr; + handles->clear(); DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname); impl->mutex_.Lock(); Status s = impl->Recover(column_families, true /* read only */, error_if_log_file_exist); + if (s.ok()) { + // set column family handles + for (auto cf : column_families) { + auto cfd = + impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name); + if (cfd == nullptr) { + s = Status::InvalidArgument("Column family not found: ", cf.name); + break; + } + handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_)); + } + } if (s.ok()) { for (auto cfd : *impl->versions_->GetColumnFamilySet()) { delete cfd->InstallSuperVersion(new SuperVersion(), &impl->mutex_); @@ -109,9 +141,14 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname, if (s.ok()) { *dbptr = impl; } else { + for (auto h : *handles) { + delete h; + } + handles->clear(); delete impl; } return s; } + } // namespace rocksdb diff --git a/db/db_test.cc b/db/db_test.cc index 0c728184a..2763cbf61 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -468,8 +468,7 @@ class DBTest { const Options* options = nullptr) { CreateColumnFamilies(cfs, options); std::vector cfs_plus_default = cfs; - cfs_plus_default.insert(cfs_plus_default.begin(), - default_column_family_name); + cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName); ReopenWithColumnFamilies(cfs_plus_default, options); } diff --git a/db/version_set.cc b/db/version_set.cc index 84361f5ff..1ac135a9b 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -1813,7 +1813,8 @@ void VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd, Builder* builder, } Status VersionSet::Recover( - const std::vector& column_families) { + const std::vector& column_families, + bool read_only) { std::unordered_map cf_name_to_options; for (auto cf : column_families) { cf_name_to_options.insert({cf.name, cf.options}); @@ -1872,12 +1873,12 @@ Status VersionSet::Recover( std::unordered_map builders; // add default column family - auto default_cf_iter = cf_name_to_options.find(default_column_family_name); + auto default_cf_iter = cf_name_to_options.find(kDefaultColumnFamilyName); if (default_cf_iter == cf_name_to_options.end()) { return Status::InvalidArgument("Default column family not specified"); } VersionEdit default_cf_edit; - default_cf_edit.AddColumnFamily(default_column_family_name); + default_cf_edit.AddColumnFamily(kDefaultColumnFamilyName); default_cf_edit.SetColumnFamily(0); ColumnFamilyData* default_cfd = CreateColumnFamily(default_cf_iter->second, &default_cf_edit); @@ -2034,11 +2035,16 @@ Status VersionSet::Recover( } // there were some column families in the MANIFEST that weren't specified - // in the argument - if (column_families_not_found.size() > 0) { + // in the argument. This is OK in read_only mode + if (read_only == false && column_families_not_found.size() > 0) { + std::string list_of_not_found; + for (auto cf : column_families_not_found) { + list_of_not_found += ", " + cf; + } + list_of_not_found = list_of_not_found.substr(2); s = Status::InvalidArgument( - "Found unexpected column families. You have to specify all column " - "families when opening the DB"); + "You have to open all column families. Column families not opened: %s", + list_of_not_found.c_str()); } if (s.ok()) { @@ -2121,7 +2127,7 @@ Status VersionSet::ListColumnFamilies(std::vector* column_families, std::map column_family_names; // default column family is always implicitly there - column_family_names.insert({0, default_column_family_name}); + column_family_names.insert({0, kDefaultColumnFamilyName}); VersionSet::LogReporter reporter; reporter.status = &s; log::Reader reader(std::move(file), &reporter, true /*checksum*/, @@ -2180,7 +2186,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname, Status status; std::vector dummy; - ColumnFamilyDescriptor dummy_descriptor(default_column_family_name, + ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName, ColumnFamilyOptions(*options)); dummy.push_back(dummy_descriptor); status = versions.Recover(dummy); @@ -2264,7 +2270,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname, // add default column family VersionEdit default_cf_edit; - default_cf_edit.AddColumnFamily(default_column_family_name); + default_cf_edit.AddColumnFamily(kDefaultColumnFamilyName); default_cf_edit.SetColumnFamily(0); ColumnFamilyData* default_cfd = CreateColumnFamily(ColumnFamilyOptions(options), &default_cf_edit); diff --git a/db/version_set.h b/db/version_set.h index d3bd97f3f..05b391dca 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -309,7 +309,10 @@ class VersionSet { nullptr); // Recover the last saved descriptor from persistent storage. - Status Recover(const std::vector& column_families); + // If read_only == true, Recover() will not complain if some column families + // are not opened + Status Recover(const std::vector& column_families, + bool read_only = false); // Reads a manifest file and returns a list of column families in // column_families. diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index 2159d35ca..89366d24f 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -28,13 +28,13 @@ class ColumnFamilyHandle { public: virtual ~ColumnFamilyHandle() {} }; -extern const std::string default_column_family_name; +extern const std::string kDefaultColumnFamilyName; struct ColumnFamilyDescriptor { std::string name; ColumnFamilyOptions options; ColumnFamilyDescriptor() - : name(default_column_family_name), options(ColumnFamilyOptions()) {} + : name(kDefaultColumnFamilyName), options(ColumnFamilyOptions()) {} ColumnFamilyDescriptor(const std::string& name, const ColumnFamilyOptions& options) : name(name), options(options) {} @@ -104,18 +104,30 @@ class DB { // that modify data, like put/delete, will return error. // If the db is opened in read only mode, then no compactions // will happen. - // TODO(icanadi): implement OpenForReadOnly that specifies column families. - // User can open DB in read-only mode even if not specifying all column - // families static Status OpenForReadOnly(const Options& options, const std::string& name, DB** dbptr, bool error_if_log_file_exist = false); + // Open the database for read only with column families. When opening DB with + // read only, you can specify only a subset of column families in the + // database that should be opened. However, you always need to specify default + // column family. The default column family name is 'default' and it's stored + // in rocksdb::kDefaultColumnFamilyName + static Status OpenForReadOnly( + const DBOptions& db_options, const std::string& name, + const std::vector& column_families, + std::vector* handles, DB** dbptr, + bool error_if_log_file_exist = false); + // Open DB with column families. // db_options specify database specific options - // column_families is the vector of all column families you'd like to open, - // containing column family name and options. The default column family name - // is 'default'. + // column_families is the vector of all column families in the databse, + // containing column family name and options. You need to open ALL column + // families in the database. To get the list of column families, you can use + // ListColumnFamilies(). Also, you can open only a subset of column families + // for read-only access. + // The default column family name is 'default' and it's stored + // in rocksdb::kDefaultColumnFamilyName. // If everything is OK, handles will on return be the same size // as column_families --- handles[i] will be a handle that you // will use to operate on column family column_family[i] diff --git a/tools/db_stress.cc b/tools/db_stress.cc index a96ee3144..5503c36f0 100644 --- a/tools/db_stress.cc +++ b/tools/db_stress.cc @@ -1519,7 +1519,7 @@ class StressTest { // DB doesn't exist assert(existing_column_families.empty()); assert(column_family_names_.empty()); - column_family_names_.push_back(default_column_family_name); + column_family_names_.push_back(kDefaultColumnFamilyName); } else if (column_family_names_.empty()) { // this is the first call to the function Open() column_family_names_ = existing_column_families; @@ -1547,7 +1547,7 @@ class StressTest { } std::vector cf_descriptors; for (auto name : column_family_names_) { - if (name != default_column_family_name) { + if (name != kDefaultColumnFamilyName) { new_column_family_name_ = std::max(new_column_family_name_.load(), std::stoi(name) + 1); } diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index 698fab36e..738a5c081 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -1068,7 +1068,7 @@ Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt, const InternalKeyComparator cmp(opt.comparator); VersionSet versions(db_path_, &opt, soptions, tc.get()); std::vector dummy; - ColumnFamilyDescriptor dummy_descriptor(default_column_family_name, + ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName, ColumnFamilyOptions(opt)); dummy.push_back(dummy_descriptor); // We rely the VersionSet::Recover to tell us the internal data structures