67f37cf198
Summary: Allow user to explicitly specify that the generated file by SstFileWriter will be ingested in a specific CF. This allow us to persist the CF id in the generated file Closes https://github.com/facebook/rocksdb/pull/1615 Differential Revision: D4270422 Pulled By: IslamAbdelRahman fbshipit-source-id: 7fb954e
146 lines
4.9 KiB
C++
146 lines
4.9 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
#pragma once
|
|
#include <string>
|
|
#include <unordered_set>
|
|
#include <vector>
|
|
|
|
#include "db/column_family.h"
|
|
#include "db/dbformat.h"
|
|
#include "db/internal_stats.h"
|
|
#include "db/snapshot_impl.h"
|
|
#include "rocksdb/db.h"
|
|
#include "rocksdb/env.h"
|
|
#include "rocksdb/sst_file_writer.h"
|
|
#include "util/autovector.h"
|
|
#include "util/db_options.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
struct IngestedFileInfo {
|
|
// External file path
|
|
std::string external_file_path;
|
|
// Smallest user key in external file
|
|
std::string smallest_user_key;
|
|
// Largest user key in external file
|
|
std::string largest_user_key;
|
|
// Sequence number for keys in external file
|
|
SequenceNumber original_seqno;
|
|
// Offset of the global sequence number field in the file, will
|
|
// be zero if version is 1 (global seqno is not supported)
|
|
size_t global_seqno_offset;
|
|
// External file size
|
|
uint64_t file_size;
|
|
// total number of keys in external file
|
|
uint64_t num_entries;
|
|
// Id of column family this file shoule be ingested into
|
|
uint32_t cf_id;
|
|
// Version of external file
|
|
int version;
|
|
|
|
// FileDescriptor for the file inside the DB
|
|
FileDescriptor fd;
|
|
// file path that we picked for file inside the DB
|
|
std::string internal_file_path = "";
|
|
// Global sequence number that we picked for the file inside the DB
|
|
SequenceNumber assigned_seqno = 0;
|
|
// Level inside the DB we picked for the external file.
|
|
int picked_level = 0;
|
|
|
|
InternalKey smallest_internal_key() const {
|
|
return InternalKey(smallest_user_key, assigned_seqno,
|
|
ValueType::kTypeValue);
|
|
}
|
|
|
|
InternalKey largest_internal_key() const {
|
|
return InternalKey(largest_user_key, assigned_seqno, ValueType::kTypeValue);
|
|
}
|
|
};
|
|
|
|
class ExternalSstFileIngestionJob {
|
|
public:
|
|
ExternalSstFileIngestionJob(
|
|
Env* env, VersionSet* versions, ColumnFamilyData* cfd,
|
|
const ImmutableDBOptions& db_options, const EnvOptions& env_options,
|
|
SnapshotList* db_snapshots,
|
|
const IngestExternalFileOptions& ingestion_options)
|
|
: env_(env),
|
|
versions_(versions),
|
|
cfd_(cfd),
|
|
db_options_(db_options),
|
|
env_options_(env_options),
|
|
db_snapshots_(db_snapshots),
|
|
ingestion_options_(ingestion_options),
|
|
job_start_time_(env_->NowMicros()) {}
|
|
|
|
// Prepare the job by copying external files into the DB.
|
|
Status Prepare(const std::vector<std::string>& external_files_paths);
|
|
|
|
// Check if we need to flush the memtable before running the ingestion job
|
|
// This will be true if the files we are ingesting are overlapping with any
|
|
// key range in the memtable.
|
|
// REQUIRES: Mutex held
|
|
Status NeedsFlush(bool* flush_needed);
|
|
|
|
// Will execute the ingestion job and prepare edit() to be applied.
|
|
// REQUIRES: Mutex held
|
|
Status Run();
|
|
|
|
// Update column family stats.
|
|
// REQUIRES: Mutex held
|
|
void UpdateStats();
|
|
|
|
// Cleanup after successfull/failed job
|
|
void Cleanup(const Status& status);
|
|
|
|
VersionEdit* edit() { return &edit_; }
|
|
|
|
private:
|
|
// Open the external file and populate `file_to_ingest` with all the
|
|
// external information we need to ingest this file.
|
|
Status GetIngestedFileInfo(const std::string& external_file,
|
|
IngestedFileInfo* file_to_ingest);
|
|
|
|
// Check if the files we are ingesting overlap with any memtable.
|
|
// REQUIRES: Mutex held
|
|
Status IngestedFilesOverlapWithMemtables(SuperVersion* sv, bool* overlap);
|
|
|
|
// Assign `file_to_ingest` the lowest possible level that it can
|
|
// be ingested to.
|
|
// REQUIRES: Mutex held
|
|
Status AssignLevelForIngestedFile(SuperVersion* sv,
|
|
IngestedFileInfo* file_to_ingest,
|
|
bool* overlap_with_db);
|
|
|
|
// Set the file global sequence number to `seqno`
|
|
Status AssignGlobalSeqnoForIngestedFile(IngestedFileInfo* file_to_ingest,
|
|
SequenceNumber seqno);
|
|
|
|
// Check if `file_to_ingest` key range overlap with the range `iter` represent
|
|
// REQUIRES: Mutex held
|
|
Status IngestedFileOverlapWithIteratorRange(
|
|
const IngestedFileInfo* file_to_ingest, InternalIterator* iter,
|
|
bool* overlap);
|
|
|
|
// Check if `file_to_ingest` can fit in level `level`
|
|
// REQUIRES: Mutex held
|
|
bool IngestedFileFitInLevel(const IngestedFileInfo* file_to_ingest,
|
|
int level);
|
|
|
|
Env* env_;
|
|
VersionSet* versions_;
|
|
ColumnFamilyData* cfd_;
|
|
const ImmutableDBOptions& db_options_;
|
|
const EnvOptions& env_options_;
|
|
SnapshotList* db_snapshots_;
|
|
autovector<IngestedFileInfo> files_to_ingest_;
|
|
const IngestExternalFileOptions& ingestion_options_;
|
|
VersionEdit edit_;
|
|
uint64_t job_start_time_;
|
|
};
|
|
|
|
} // namespace rocksdb
|