2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2017-07-16 01:03:42 +02:00
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
2014-11-21 00:54:47 +01:00
|
|
|
//
|
|
|
|
// Copyright (c) 2012 Facebook.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file.
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
2017-04-24 23:57:27 +02:00
|
|
|
#include "utilities/checkpoint/checkpoint_impl.h"
|
2014-11-21 00:54:47 +01:00
|
|
|
|
|
|
|
#include <algorithm>
|
2019-09-20 21:00:55 +02:00
|
|
|
#include <cinttypes>
|
2014-11-21 00:54:47 +01:00
|
|
|
#include <string>
|
2021-02-17 21:40:33 +01:00
|
|
|
#include <tuple>
|
2021-10-16 19:03:19 +02:00
|
|
|
#include <unordered_set>
|
2017-04-24 23:57:27 +02:00
|
|
|
#include <vector>
|
|
|
|
|
2015-06-20 01:08:31 +02:00
|
|
|
#include "db/wal_manager.h"
|
2019-05-30 05:44:08 +02:00
|
|
|
#include "file/file_util.h"
|
|
|
|
#include "file/filename.h"
|
2021-09-29 13:01:57 +02:00
|
|
|
#include "logging/logging.h"
|
2017-04-04 03:27:24 +02:00
|
|
|
#include "port/port.h"
|
2014-11-21 00:54:47 +01:00
|
|
|
#include "rocksdb/db.h"
|
|
|
|
#include "rocksdb/env.h"
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
#include "rocksdb/metadata.h"
|
2021-10-16 19:03:19 +02:00
|
|
|
#include "rocksdb/options.h"
|
2015-06-20 01:08:31 +02:00
|
|
|
#include "rocksdb/transaction_log.h"
|
2021-10-16 19:03:19 +02:00
|
|
|
#include "rocksdb/types.h"
|
2017-04-24 23:57:27 +02:00
|
|
|
#include "rocksdb/utilities/checkpoint.h"
|
2019-05-30 20:21:38 +02:00
|
|
|
#include "test_util/sync_point.h"
|
2020-07-03 04:24:25 +02:00
|
|
|
#include "util/cast_util.h"
|
2020-08-26 19:37:59 +02:00
|
|
|
#include "util/file_checksum_helper.h"
|
2014-11-21 00:54:47 +01:00
|
|
|
|
2020-02-20 21:07:53 +01:00
|
|
|
namespace ROCKSDB_NAMESPACE {
|
2014-11-21 00:54:47 +01:00
|
|
|
|
|
|
|
Status Checkpoint::Create(DB* db, Checkpoint** checkpoint_ptr) {
|
|
|
|
*checkpoint_ptr = new CheckpointImpl(db);
|
|
|
|
return Status::OK();
|
|
|
|
}
|
|
|
|
|
2018-03-05 22:08:17 +01:00
|
|
|
Status Checkpoint::CreateCheckpoint(const std::string& /*checkpoint_dir*/,
|
2020-03-10 21:37:37 +01:00
|
|
|
uint64_t /*log_size_for_flush*/,
|
2021-08-03 03:27:11 +02:00
|
|
|
uint64_t* /*sequence_number_ptr*/) {
|
2014-11-21 00:54:47 +01:00
|
|
|
return Status::NotSupported("");
|
|
|
|
}
|
|
|
|
|
2021-08-03 03:27:11 +02:00
|
|
|
void CheckpointImpl::CleanStagingDirectory(const std::string& full_private_path,
|
|
|
|
Logger* info_log) {
|
|
|
|
std::vector<std::string> subchildren;
|
2018-06-22 01:22:57 +02:00
|
|
|
Status s = db_->GetEnv()->FileExists(full_private_path);
|
|
|
|
if (s.IsNotFound()) {
|
|
|
|
return;
|
|
|
|
}
|
2021-08-03 03:27:11 +02:00
|
|
|
ROCKS_LOG_INFO(info_log, "File exists %s -- %s", full_private_path.c_str(),
|
|
|
|
s.ToString().c_str());
|
2020-12-10 06:19:55 +01:00
|
|
|
s = db_->GetEnv()->GetChildren(full_private_path, &subchildren);
|
|
|
|
if (s.ok()) {
|
|
|
|
for (auto& subchild : subchildren) {
|
|
|
|
std::string subchild_path = full_private_path + "/" + subchild;
|
|
|
|
s = db_->GetEnv()->DeleteFile(subchild_path);
|
|
|
|
ROCKS_LOG_INFO(info_log, "Delete file %s -- %s", subchild_path.c_str(),
|
|
|
|
s.ToString().c_str());
|
|
|
|
}
|
2018-06-22 01:22:57 +02:00
|
|
|
}
|
|
|
|
// finally delete the private dir
|
|
|
|
s = db_->GetEnv()->DeleteDir(full_private_path);
|
2021-08-03 03:27:11 +02:00
|
|
|
ROCKS_LOG_INFO(info_log, "Delete dir %s -- %s", full_private_path.c_str(),
|
|
|
|
s.ToString().c_str());
|
2018-06-22 01:22:57 +02:00
|
|
|
}
|
|
|
|
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
Status Checkpoint::ExportColumnFamily(
|
|
|
|
ColumnFamilyHandle* /*handle*/, const std::string& /*export_dir*/,
|
|
|
|
ExportImportFilesMetaData** /*metadata*/) {
|
|
|
|
return Status::NotSupported("");
|
|
|
|
}
|
|
|
|
|
2014-11-21 00:54:47 +01:00
|
|
|
// Builds an openable snapshot of RocksDB
|
2017-03-22 01:53:21 +01:00
|
|
|
Status CheckpointImpl::CreateCheckpoint(const std::string& checkpoint_dir,
|
2020-03-10 21:37:37 +01:00
|
|
|
uint64_t log_size_for_flush,
|
2021-08-03 03:27:11 +02:00
|
|
|
uint64_t* sequence_number_ptr) {
|
2016-12-29 03:38:20 +01:00
|
|
|
DBOptions db_options = db_->GetDBOptions();
|
2014-11-21 00:54:47 +01:00
|
|
|
|
2017-04-24 23:57:27 +02:00
|
|
|
Status s = db_->GetEnv()->FileExists(checkpoint_dir);
|
2015-07-21 02:20:40 +02:00
|
|
|
if (s.ok()) {
|
2014-11-21 00:54:47 +01:00
|
|
|
return Status::InvalidArgument("Directory exists");
|
2015-07-21 02:20:40 +02:00
|
|
|
} else if (!s.IsNotFound()) {
|
|
|
|
assert(s.IsIOError());
|
|
|
|
return s;
|
2014-11-21 00:54:47 +01:00
|
|
|
}
|
|
|
|
|
2017-04-24 23:57:27 +02:00
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
db_options.info_log,
|
|
|
|
"Started the snapshot process -- creating snapshot in directory %s",
|
|
|
|
checkpoint_dir.c_str());
|
2018-01-30 05:59:59 +01:00
|
|
|
|
|
|
|
size_t final_nonslash_idx = checkpoint_dir.find_last_not_of('/');
|
|
|
|
if (final_nonslash_idx == std::string::npos) {
|
2018-02-21 01:42:06 +01:00
|
|
|
// npos means it's only slashes or empty. Non-empty means it's the root
|
|
|
|
// directory, but it shouldn't be because we verified above the directory
|
|
|
|
// doesn't exist.
|
|
|
|
assert(checkpoint_dir.empty());
|
2018-01-30 05:59:59 +01:00
|
|
|
return Status::InvalidArgument("invalid checkpoint directory name");
|
|
|
|
}
|
|
|
|
|
2021-08-03 03:27:11 +02:00
|
|
|
std::string full_private_path =
|
|
|
|
checkpoint_dir.substr(0, final_nonslash_idx + 1) + ".tmp";
|
Checkpoint dir options fix (#8572)
Summary:
Originally the 2 options `db_log_dir` and `wal_dir` will be reused in a snapshot db since the options files are just copied. By default, if `wal_dir` was not set when a db was created, it is set to the db's dir. Therefore, the snapshot db will use the same WAL dir. If both the original db and the snapshot db write to or delete from the WAL dir, one may modify or delete files which belong to the other. The same applies to `db_log_dir` as well, but as info log files are not copied or linked, it is simpler for this option.
2 arguments are added to `Checkpoint::CreateCheckpoint()`, allowing to override these 2 options.
`wal_dir`: If the function argument `wal_dir` is empty, or set to the original db location, or the checkpoint location, the snapshot's `wal_dir` option will be updated to the checkpoint location. Otherwise, the absolute path specified in the argument will be used. During checkpointing, live WAL files will be copied or linked the new location, instead of the current WAL dir specified in the original db.
`db_log_dir`: Same as `wal_dir`, but no files will be copied or linked.
A new unit test was added: `CheckpointTest.CheckpointWithOptionsDirsTest`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8572
Test Plan:
New unit test
```
checkpoint_test --gtest_filter="CheckpointTest.CheckpointWithOptionsDirsTest"
```
Output
```
Note: Google Test filter = CheckpointTest.CheckpointWithOptionsDirsTest
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
[----------] 1 test from CheckpointTest
[ RUN ] CheckpointTest.CheckpointWithOptionsDirsTest
[ OK ] CheckpointTest.CheckpointWithOptionsDirsTest (11712 ms)
[----------] 1 test from CheckpointTest (11712 ms total)
[----------] Global test environment tear-down
[==========] 1 test from 1 test case ran. (11713 ms total)
[ PASSED ] 1 test.
```
This test will fail without this patch. Just modify the code to remove the 2 arguments introduced in this patch in `CreateCheckpoint()`.
Reviewed By: zhichao-cao
Differential Revision: D29832761
Pulled By: autopear
fbshipit-source-id: e6a639b4d674380df82998c0839e79cab695fe29
2021-07-23 20:11:25 +02:00
|
|
|
ROCKS_LOG_INFO(db_options.info_log,
|
|
|
|
"Snapshot process -- using temporary directory %s",
|
|
|
|
full_private_path.c_str());
|
2018-06-22 01:22:57 +02:00
|
|
|
CleanStagingDirectory(full_private_path, db_options.info_log.get());
|
2017-04-24 23:57:27 +02:00
|
|
|
// create snapshot directory
|
|
|
|
s = db_->GetEnv()->CreateDir(full_private_path);
|
|
|
|
uint64_t sequence_number = 0;
|
|
|
|
if (s.ok()) {
|
2020-12-10 06:19:55 +01:00
|
|
|
// enable file deletions
|
|
|
|
s = db_->DisableFileDeletions();
|
|
|
|
const bool disabled_file_deletions = s.ok();
|
|
|
|
|
|
|
|
if (s.ok() || s.IsNotSupported()) {
|
|
|
|
s = CreateCustomCheckpoint(
|
|
|
|
[&](const std::string& src_dirname, const std::string& fname,
|
2021-08-03 03:27:11 +02:00
|
|
|
FileType) {
|
2020-12-10 06:19:55 +01:00
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Hard Linking %s",
|
|
|
|
fname.c_str());
|
2021-10-16 19:03:19 +02:00
|
|
|
return db_->GetFileSystem()->LinkFile(
|
|
|
|
src_dirname + "/" + fname, full_private_path + "/" + fname,
|
|
|
|
IOOptions(), nullptr);
|
2020-12-10 06:19:55 +01:00
|
|
|
} /* link_file_cb */,
|
|
|
|
[&](const std::string& src_dirname, const std::string& fname,
|
2021-08-03 03:27:11 +02:00
|
|
|
uint64_t size_limit_bytes, FileType,
|
2020-12-10 06:19:55 +01:00
|
|
|
const std::string& /* checksum_func_name */,
|
2022-02-19 03:18:49 +01:00
|
|
|
const std::string& /* checksum_val */,
|
|
|
|
const Temperature temperature) {
|
2020-12-10 06:19:55 +01:00
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Copying %s", fname.c_str());
|
2021-10-16 19:03:19 +02:00
|
|
|
return CopyFile(db_->GetFileSystem(), src_dirname + "/" + fname,
|
|
|
|
full_private_path + "/" + fname, size_limit_bytes,
|
2022-02-19 03:18:49 +01:00
|
|
|
db_options.use_fsync, nullptr, temperature);
|
2020-12-10 06:19:55 +01:00
|
|
|
} /* copy_file_cb */,
|
|
|
|
[&](const std::string& fname, const std::string& contents, FileType) {
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Creating %s", fname.c_str());
|
2021-10-16 19:03:19 +02:00
|
|
|
return CreateFile(db_->GetFileSystem(),
|
|
|
|
full_private_path + "/" + fname, contents,
|
|
|
|
db_options.use_fsync);
|
2020-12-10 06:19:55 +01:00
|
|
|
} /* create_file_cb */,
|
|
|
|
&sequence_number, log_size_for_flush);
|
|
|
|
|
|
|
|
// we copied all the files, enable file deletions
|
|
|
|
if (disabled_file_deletions) {
|
|
|
|
Status ss = db_->EnableFileDeletions(false);
|
|
|
|
assert(ss.ok());
|
|
|
|
ss.PermitUncheckedError();
|
|
|
|
}
|
|
|
|
}
|
2017-04-24 23:57:27 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
// move tmp private backup to real snapshot directory
|
2021-08-03 03:27:11 +02:00
|
|
|
s = db_->GetEnv()->RenameFile(full_private_path, checkpoint_dir);
|
2017-04-24 23:57:27 +02:00
|
|
|
}
|
|
|
|
if (s.ok()) {
|
2021-11-03 20:20:19 +01:00
|
|
|
std::unique_ptr<FSDirectory> checkpoint_directory;
|
|
|
|
s = db_->GetFileSystem()->NewDirectory(checkpoint_dir, IOOptions(),
|
|
|
|
&checkpoint_directory, nullptr);
|
2020-12-10 06:19:55 +01:00
|
|
|
if (s.ok() && checkpoint_directory != nullptr) {
|
2021-11-03 20:20:19 +01:00
|
|
|
s = checkpoint_directory->FsyncWithDirOptions(
|
|
|
|
IOOptions(), nullptr,
|
|
|
|
DirFsyncOptions(DirFsyncOptions::FsyncReason::kDirRenamed));
|
2017-04-24 23:57:27 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
2020-03-10 21:37:37 +01:00
|
|
|
if (sequence_number_ptr != nullptr) {
|
|
|
|
*sequence_number_ptr = sequence_number;
|
|
|
|
}
|
2017-04-24 23:57:27 +02:00
|
|
|
// here we know that we succeeded and installed the new snapshot
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Snapshot DONE. All is good");
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Snapshot sequence number: %" PRIu64,
|
|
|
|
sequence_number);
|
|
|
|
} else {
|
|
|
|
// clean all the files we might have created
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Snapshot failed -- %s",
|
|
|
|
s.ToString().c_str());
|
2018-06-22 01:22:57 +02:00
|
|
|
CleanStagingDirectory(full_private_path, db_options.info_log.get());
|
2017-04-24 23:57:27 +02:00
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status CheckpointImpl::CreateCustomCheckpoint(
|
|
|
|
std::function<Status(const std::string& src_dirname,
|
|
|
|
const std::string& src_fname, FileType type)>
|
|
|
|
link_file_cb,
|
2022-02-19 03:18:49 +01:00
|
|
|
std::function<
|
|
|
|
Status(const std::string& src_dirname, const std::string& src_fname,
|
|
|
|
uint64_t size_limit_bytes, FileType type,
|
|
|
|
const std::string& checksum_func_name,
|
|
|
|
const std::string& checksum_val, const Temperature temperature)>
|
2017-04-24 23:57:27 +02:00
|
|
|
copy_file_cb,
|
|
|
|
std::function<Status(const std::string& fname, const std::string& contents,
|
|
|
|
FileType type)>
|
|
|
|
create_file_cb,
|
2020-07-03 03:13:31 +02:00
|
|
|
uint64_t* sequence_number, uint64_t log_size_for_flush,
|
2020-08-26 19:37:59 +02:00
|
|
|
bool get_live_table_checksum) {
|
2017-04-24 23:57:27 +02:00
|
|
|
*sequence_number = db_->GetLatestSequenceNumber();
|
2020-12-10 06:19:55 +01:00
|
|
|
|
2021-10-16 19:03:19 +02:00
|
|
|
LiveFilesStorageInfoOptions opts;
|
|
|
|
opts.include_checksum_info = get_live_table_checksum;
|
|
|
|
opts.wal_size_for_flush = log_size_for_flush;
|
2016-12-28 20:53:29 +01:00
|
|
|
|
2021-10-16 19:03:19 +02:00
|
|
|
std::vector<LiveFileStorageInfo> infos;
|
|
|
|
{
|
|
|
|
Status s = db_->GetLiveFilesStorageInfo(opts, &infos);
|
2020-08-26 19:37:59 +02:00
|
|
|
if (!s.ok()) {
|
2021-10-16 19:03:19 +02:00
|
|
|
return s;
|
2020-08-26 19:37:59 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-16 19:03:19 +02:00
|
|
|
// Verify that everything except WAL files are in same directory
|
|
|
|
// (db_paths / cf_paths not supported)
|
|
|
|
std::unordered_set<std::string> dirs;
|
|
|
|
for (auto& info : infos) {
|
|
|
|
if (info.file_type != kWalFile) {
|
|
|
|
dirs.insert(info.directory);
|
2014-11-21 00:54:47 +01:00
|
|
|
}
|
|
|
|
}
|
2021-10-16 19:03:19 +02:00
|
|
|
if (dirs.size() > 1) {
|
|
|
|
return Status::NotSupported(
|
|
|
|
"db_paths / cf_paths not supported for Checkpoint nor BackupEngine");
|
2016-03-17 18:07:21 +01:00
|
|
|
}
|
2015-06-20 01:08:31 +02:00
|
|
|
|
2021-10-16 19:03:19 +02:00
|
|
|
bool same_fs = true;
|
|
|
|
|
|
|
|
for (auto& info : infos) {
|
|
|
|
Status s;
|
|
|
|
if (!info.replacement_contents.empty()) {
|
|
|
|
// Currently should only be used for CURRENT file.
|
|
|
|
assert(info.file_type == kCurrentFile);
|
|
|
|
|
|
|
|
if (info.size != info.replacement_contents.size()) {
|
|
|
|
s = Status::Corruption("Inconsistent size metadata for " +
|
|
|
|
info.relative_filename);
|
|
|
|
} else {
|
|
|
|
s = create_file_cb(info.relative_filename, info.replacement_contents,
|
|
|
|
info.file_type);
|
2015-06-20 01:08:31 +02:00
|
|
|
}
|
2021-10-16 19:03:19 +02:00
|
|
|
} else {
|
|
|
|
if (same_fs && !info.trim_to_size) {
|
|
|
|
s = link_file_cb(info.directory, info.relative_filename,
|
|
|
|
info.file_type);
|
2015-06-20 01:08:31 +02:00
|
|
|
if (s.IsNotSupported()) {
|
|
|
|
same_fs = false;
|
|
|
|
s = Status::OK();
|
|
|
|
}
|
2021-10-16 19:03:19 +02:00
|
|
|
s.MustCheck();
|
2015-06-20 01:08:31 +02:00
|
|
|
}
|
2021-10-16 19:03:19 +02:00
|
|
|
if (!same_fs || info.trim_to_size) {
|
|
|
|
assert(info.file_checksum_func_name.empty() ==
|
|
|
|
!opts.include_checksum_info);
|
|
|
|
// no assertion on file_checksum because empty is used for both "not
|
|
|
|
// set" and "unknown"
|
|
|
|
if (opts.include_checksum_info) {
|
|
|
|
s = copy_file_cb(info.directory, info.relative_filename, info.size,
|
|
|
|
info.file_type, info.file_checksum_func_name,
|
2022-02-19 03:18:49 +01:00
|
|
|
info.file_checksum, info.temperature);
|
2021-10-16 19:03:19 +02:00
|
|
|
} else {
|
|
|
|
s = copy_file_cb(info.directory, info.relative_filename, info.size,
|
|
|
|
info.file_type, kUnknownFileChecksumFuncName,
|
2022-02-19 03:18:49 +01:00
|
|
|
kUnknownFileChecksum, info.temperature);
|
2021-10-16 19:03:19 +02:00
|
|
|
}
|
2015-06-20 01:08:31 +02:00
|
|
|
}
|
|
|
|
}
|
2021-10-16 19:03:19 +02:00
|
|
|
if (!s.ok()) {
|
|
|
|
return s;
|
|
|
|
}
|
2015-06-20 01:08:31 +02:00
|
|
|
}
|
2014-11-21 00:54:47 +01:00
|
|
|
|
2021-10-16 19:03:19 +02:00
|
|
|
return Status::OK();
|
2014-11-21 00:54:47 +01:00
|
|
|
}
|
2017-04-24 23:57:27 +02:00
|
|
|
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
// Exports all live SST files of a specified Column Family onto export_dir,
|
|
|
|
// returning SST files information in metadata.
|
|
|
|
Status CheckpointImpl::ExportColumnFamily(
|
|
|
|
ColumnFamilyHandle* handle, const std::string& export_dir,
|
|
|
|
ExportImportFilesMetaData** metadata) {
|
2020-07-03 04:24:25 +02:00
|
|
|
auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(handle);
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
const auto cf_name = cfh->GetName();
|
|
|
|
const auto db_options = db_->GetDBOptions();
|
|
|
|
|
|
|
|
assert(metadata != nullptr);
|
|
|
|
assert(*metadata == nullptr);
|
|
|
|
auto s = db_->GetEnv()->FileExists(export_dir);
|
|
|
|
if (s.ok()) {
|
|
|
|
return Status::InvalidArgument("Specified export_dir exists");
|
|
|
|
} else if (!s.IsNotFound()) {
|
|
|
|
assert(s.IsIOError());
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
const auto final_nonslash_idx = export_dir.find_last_not_of('/');
|
|
|
|
if (final_nonslash_idx == std::string::npos) {
|
|
|
|
return Status::InvalidArgument("Specified export_dir invalid");
|
|
|
|
}
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log,
|
|
|
|
"[%s] export column family onto export directory %s",
|
|
|
|
cf_name.c_str(), export_dir.c_str());
|
|
|
|
|
|
|
|
// Create a temporary export directory.
|
|
|
|
const auto tmp_export_dir =
|
|
|
|
export_dir.substr(0, final_nonslash_idx + 1) + ".tmp";
|
|
|
|
s = db_->GetEnv()->CreateDir(tmp_export_dir);
|
|
|
|
|
|
|
|
if (s.ok()) {
|
2020-02-20 21:07:53 +01:00
|
|
|
s = db_->Flush(ROCKSDB_NAMESPACE::FlushOptions(), handle);
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
ColumnFamilyMetaData db_metadata;
|
|
|
|
if (s.ok()) {
|
|
|
|
// Export live sst files with file deletions disabled.
|
|
|
|
s = db_->DisableFileDeletions();
|
|
|
|
if (s.ok()) {
|
|
|
|
db_->GetColumnFamilyMetaData(handle, &db_metadata);
|
|
|
|
|
|
|
|
s = ExportFilesInMetaData(
|
|
|
|
db_options, db_metadata,
|
|
|
|
[&](const std::string& src_dirname, const std::string& fname) {
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "[%s] HardLinking %s",
|
|
|
|
cf_name.c_str(), fname.c_str());
|
|
|
|
return db_->GetEnv()->LinkFile(src_dirname + fname,
|
|
|
|
tmp_export_dir + fname);
|
|
|
|
} /*link_file_cb*/,
|
|
|
|
[&](const std::string& src_dirname, const std::string& fname) {
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "[%s] Copying %s",
|
|
|
|
cf_name.c_str(), fname.c_str());
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
2019-12-13 23:47:08 +01:00
|
|
|
return CopyFile(db_->GetFileSystem(), src_dirname + fname,
|
2022-02-19 03:18:49 +01:00
|
|
|
tmp_export_dir + fname, 0, db_options.use_fsync,
|
|
|
|
nullptr, Temperature::kUnknown);
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
} /*copy_file_cb*/);
|
|
|
|
|
|
|
|
const auto enable_status = db_->EnableFileDeletions(false /*force*/);
|
|
|
|
if (s.ok()) {
|
|
|
|
s = enable_status;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
auto moved_to_user_specified_dir = false;
|
|
|
|
if (s.ok()) {
|
|
|
|
// Move temporary export directory to the actual export directory.
|
|
|
|
s = db_->GetEnv()->RenameFile(tmp_export_dir, export_dir);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
// Fsync export directory.
|
|
|
|
moved_to_user_specified_dir = true;
|
2021-11-03 20:20:19 +01:00
|
|
|
std::unique_ptr<FSDirectory> dir_ptr;
|
|
|
|
s = db_->GetFileSystem()->NewDirectory(export_dir, IOOptions(), &dir_ptr,
|
|
|
|
nullptr);
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
if (s.ok()) {
|
|
|
|
assert(dir_ptr != nullptr);
|
2021-11-03 20:20:19 +01:00
|
|
|
s = dir_ptr->FsyncWithDirOptions(
|
|
|
|
IOOptions(), nullptr,
|
|
|
|
DirFsyncOptions(DirFsyncOptions::FsyncReason::kDirRenamed));
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (s.ok()) {
|
|
|
|
// Export of files succeeded. Fill in the metadata information.
|
|
|
|
auto result_metadata = new ExportImportFilesMetaData();
|
|
|
|
result_metadata->db_comparator_name = handle->GetComparator()->Name();
|
|
|
|
for (const auto& level_metadata : db_metadata.levels) {
|
|
|
|
for (const auto& file_metadata : level_metadata.files) {
|
|
|
|
LiveFileMetaData live_file_metadata;
|
|
|
|
live_file_metadata.size = file_metadata.size;
|
|
|
|
live_file_metadata.name = std::move(file_metadata.name);
|
2019-11-07 23:02:16 +01:00
|
|
|
live_file_metadata.file_number = file_metadata.file_number;
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
live_file_metadata.db_path = export_dir;
|
|
|
|
live_file_metadata.smallest_seqno = file_metadata.smallest_seqno;
|
|
|
|
live_file_metadata.largest_seqno = file_metadata.largest_seqno;
|
|
|
|
live_file_metadata.smallestkey = std::move(file_metadata.smallestkey);
|
|
|
|
live_file_metadata.largestkey = std::move(file_metadata.largestkey);
|
2019-11-07 23:02:16 +01:00
|
|
|
live_file_metadata.oldest_blob_file_number =
|
|
|
|
file_metadata.oldest_blob_file_number;
|
Export Import sst files (#5495)
Summary:
Refresh of the earlier change here - https://github.com/facebook/rocksdb/issues/5135
This is a review request for code change needed for - https://github.com/facebook/rocksdb/issues/3469
"Add support for taking snapshot of a column family and creating column family from a given CF snapshot"
We have an implementation for this that we have been testing internally. We have two new APIs that together provide this functionality.
(1) ExportColumnFamily() - This API is modelled after CreateCheckpoint() as below.
// Exports all live SST files of a specified Column Family onto export_dir,
// returning SST files information in metadata.
// - SST files will be created as hard links when the directory specified
// is in the same partition as the db directory, copied otherwise.
// - export_dir should not already exist and will be created by this API.
// - Always triggers a flush.
virtual Status ExportColumnFamily(ColumnFamilyHandle* handle,
const std::string& export_dir,
ExportImportFilesMetaData** metadata);
Internally, the API will DisableFileDeletions(), GetColumnFamilyMetaData(), Parse through
metadata, creating links/copies of all the sst files, EnableFileDeletions() and complete the call by
returning the list of file metadata.
(2) CreateColumnFamilyWithImport() - This API is modeled after IngestExternalFile(), but invoked only during a CF creation as below.
// CreateColumnFamilyWithImport() will create a new column family with
// column_family_name and import external SST files specified in metadata into
// this column family.
// (1) External SST files can be created using SstFileWriter.
// (2) External SST files can be exported from a particular column family in
// an existing DB.
// Option in import_options specifies whether the external files are copied or
// moved (default is copy). When option specifies copy, managing files at
// external_file_path is caller's responsibility. When option specifies a
// move, the call ensures that the specified files at external_file_path are
// deleted on successful return and files are not modified on any error
// return.
// On error return, column family handle returned will be nullptr.
// ColumnFamily will be present on successful return and will not be present
// on error return. ColumnFamily may be present on any crash during this call.
virtual Status CreateColumnFamilyWithImport(
const ColumnFamilyOptions& options, const std::string& column_family_name,
const ImportColumnFamilyOptions& import_options,
const ExportImportFilesMetaData& metadata,
ColumnFamilyHandle** handle);
Internally, this API creates a new CF, parses all the sst files and adds it to the specified column family, at the same level and with same sequence number as in the metadata. Also performs safety checks with respect to overlaps between the sst files being imported.
If incoming sequence number is higher than current local sequence number, local sequence
number is updated to reflect this.
Note, as the sst files is are being moved across Column Families, Column Family name in sst file
will no longer match the actual column family on destination DB. The API does not modify Column
Family name or id in the sst files being imported.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5495
Differential Revision: D16018881
fbshipit-source-id: 9ae2251025d5916d35a9fc4ea4d6707f6be16ff9
2019-07-17 21:22:21 +02:00
|
|
|
live_file_metadata.level = level_metadata.level;
|
|
|
|
result_metadata->files.push_back(live_file_metadata);
|
|
|
|
}
|
|
|
|
*metadata = result_metadata;
|
|
|
|
}
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "[%s] Export succeeded.",
|
|
|
|
cf_name.c_str());
|
|
|
|
} else {
|
|
|
|
// Failure: Clean up all the files/directories created.
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "[%s] Export failed. %s",
|
|
|
|
cf_name.c_str(), s.ToString().c_str());
|
|
|
|
std::vector<std::string> subchildren;
|
|
|
|
const auto cleanup_dir =
|
|
|
|
moved_to_user_specified_dir ? export_dir : tmp_export_dir;
|
|
|
|
db_->GetEnv()->GetChildren(cleanup_dir, &subchildren);
|
|
|
|
for (const auto& subchild : subchildren) {
|
|
|
|
const auto subchild_path = cleanup_dir + "/" + subchild;
|
|
|
|
const auto status = db_->GetEnv()->DeleteFile(subchild_path);
|
|
|
|
if (!status.ok()) {
|
|
|
|
ROCKS_LOG_WARN(db_options.info_log, "Failed to cleanup file %s: %s",
|
|
|
|
subchild_path.c_str(), status.ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const auto status = db_->GetEnv()->DeleteDir(cleanup_dir);
|
|
|
|
if (!status.ok()) {
|
|
|
|
ROCKS_LOG_WARN(db_options.info_log, "Failed to cleanup dir %s: %s",
|
|
|
|
cleanup_dir.c_str(), status.ToString().c_str());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return s;
|
|
|
|
}
|
|
|
|
|
|
|
|
Status CheckpointImpl::ExportFilesInMetaData(
|
|
|
|
const DBOptions& db_options, const ColumnFamilyMetaData& metadata,
|
|
|
|
std::function<Status(const std::string& src_dirname,
|
|
|
|
const std::string& src_fname)>
|
|
|
|
link_file_cb,
|
|
|
|
std::function<Status(const std::string& src_dirname,
|
|
|
|
const std::string& src_fname)>
|
|
|
|
copy_file_cb) {
|
|
|
|
Status s;
|
|
|
|
auto hardlink_file = true;
|
|
|
|
|
|
|
|
// Copy/hard link files in metadata.
|
|
|
|
size_t num_files = 0;
|
|
|
|
for (const auto& level_metadata : metadata.levels) {
|
|
|
|
for (const auto& file_metadata : level_metadata.files) {
|
|
|
|
uint64_t number;
|
|
|
|
FileType type;
|
|
|
|
const auto ok = ParseFileName(file_metadata.name, &number, &type);
|
|
|
|
if (!ok) {
|
|
|
|
s = Status::Corruption("Could not parse file name");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We should only get sst files here.
|
|
|
|
assert(type == kTableFile);
|
|
|
|
assert(file_metadata.size > 0 && file_metadata.name[0] == '/');
|
|
|
|
const auto src_fname = file_metadata.name;
|
|
|
|
++num_files;
|
|
|
|
|
|
|
|
if (hardlink_file) {
|
|
|
|
s = link_file_cb(db_->GetName(), src_fname);
|
|
|
|
if (num_files == 1 && s.IsNotSupported()) {
|
|
|
|
// Fallback to copy if link failed due to cross-device directories.
|
|
|
|
hardlink_file = false;
|
|
|
|
s = Status::OK();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!hardlink_file) {
|
|
|
|
s = copy_file_cb(db_->GetName(), src_fname);
|
|
|
|
}
|
|
|
|
if (!s.ok()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
ROCKS_LOG_INFO(db_options.info_log, "Number of table files %" ROCKSDB_PRIszt,
|
|
|
|
num_files);
|
|
|
|
|
|
|
|
return s;
|
|
|
|
}
|
2020-02-20 21:07:53 +01:00
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
2014-11-21 00:54:47 +01:00
|
|
|
|
|
|
|
#endif // ROCKSDB_LITE
|