843d2e3137
Summary: This adds a new metablock containing a shared dictionary that is used to compress all data blocks in the SST file. The size of the shared dictionary is configurable in CompressionOptions and defaults to 0. It's currently only used for zlib/lz4/lz4hc, but the block will be stored in the SST regardless of the compression type if the user chooses a nonzero dictionary size. During compaction, computes the dictionary by randomly sampling the first output file in each subcompaction. It pre-computes the intervals to sample by assuming the output file will have the maximum allowable length. In case the file is smaller, some of the pre-computed sampling intervals can be beyond end-of-file, in which case we skip over those samples and the dictionary will be a bit smaller. After the dictionary is generated using the first file in a subcompaction, it is loaded into the compression library before writing each block in each subsequent file of that subcompaction. On the read path, gets the dictionary from the metablock, if it exists. Then, loads that dictionary into the compression library before reading each block. Test Plan: new unit test Reviewers: yhchiang, IslamAbdelRahman, cyan, sdong Reviewed By: sdong Subscribers: andrewkr, yoshinorim, kradhakrishnan, dhruba, leveldb Differential Revision: https://reviews.facebook.net/D52287
124 lines
4.6 KiB
C++
124 lines
4.6 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under the BSD-style license found in the
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
//
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
#pragma once
|
|
|
|
#include <stdint.h>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <vector>
|
|
#include "db/table_properties_collector.h"
|
|
#include "rocksdb/options.h"
|
|
#include "rocksdb/table_properties.h"
|
|
#include "util/file_reader_writer.h"
|
|
#include "util/mutable_cf_options.h"
|
|
|
|
namespace rocksdb {
|
|
|
|
class Slice;
|
|
class Status;
|
|
|
|
struct TableReaderOptions {
|
|
// @param skip_filters Disables loading/accessing the filter block
|
|
TableReaderOptions(const ImmutableCFOptions& _ioptions,
|
|
const EnvOptions& _env_options,
|
|
const InternalKeyComparator& _internal_comparator,
|
|
bool _skip_filters = false, int _level = -1)
|
|
: ioptions(_ioptions),
|
|
env_options(_env_options),
|
|
internal_comparator(_internal_comparator),
|
|
skip_filters(_skip_filters),
|
|
level(_level) {}
|
|
|
|
const ImmutableCFOptions& ioptions;
|
|
const EnvOptions& env_options;
|
|
const InternalKeyComparator& internal_comparator;
|
|
// This is only used for BlockBasedTable (reader)
|
|
bool skip_filters;
|
|
// what level this table/file is on, -1 for "not set, don't know"
|
|
int level;
|
|
};
|
|
|
|
struct TableBuilderOptions {
|
|
TableBuilderOptions(
|
|
const ImmutableCFOptions& _ioptions,
|
|
const InternalKeyComparator& _internal_comparator,
|
|
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
|
_int_tbl_prop_collector_factories,
|
|
CompressionType _compression_type,
|
|
const CompressionOptions& _compression_opts,
|
|
const std::string* _compression_dict, bool _skip_filters,
|
|
const std::string& _column_family_name)
|
|
: ioptions(_ioptions),
|
|
internal_comparator(_internal_comparator),
|
|
int_tbl_prop_collector_factories(_int_tbl_prop_collector_factories),
|
|
compression_type(_compression_type),
|
|
compression_opts(_compression_opts),
|
|
compression_dict(_compression_dict),
|
|
skip_filters(_skip_filters),
|
|
column_family_name(_column_family_name) {}
|
|
const ImmutableCFOptions& ioptions;
|
|
const InternalKeyComparator& internal_comparator;
|
|
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
|
int_tbl_prop_collector_factories;
|
|
CompressionType compression_type;
|
|
const CompressionOptions& compression_opts;
|
|
// Data for presetting the compression library's dictionary, or nullptr.
|
|
const std::string* compression_dict;
|
|
bool skip_filters; // only used by BlockBasedTableBuilder
|
|
const std::string& column_family_name;
|
|
};
|
|
|
|
// TableBuilder provides the interface used to build a Table
|
|
// (an immutable and sorted map from keys to values).
|
|
//
|
|
// Multiple threads can invoke const methods on a TableBuilder without
|
|
// external synchronization, but if any of the threads may call a
|
|
// non-const method, all threads accessing the same TableBuilder must use
|
|
// external synchronization.
|
|
class TableBuilder {
|
|
public:
|
|
// REQUIRES: Either Finish() or Abandon() has been called.
|
|
virtual ~TableBuilder() {}
|
|
|
|
// Add key,value to the table being constructed.
|
|
// REQUIRES: key is after any previously added key according to comparator.
|
|
// REQUIRES: Finish(), Abandon() have not been called
|
|
virtual void Add(const Slice& key, const Slice& value) = 0;
|
|
|
|
// Return non-ok iff some error has been detected.
|
|
virtual Status status() const = 0;
|
|
|
|
// Finish building the table.
|
|
// REQUIRES: Finish(), Abandon() have not been called
|
|
virtual Status Finish() = 0;
|
|
|
|
// Indicate that the contents of this builder should be abandoned.
|
|
// If the caller is not going to call Finish(), it must call Abandon()
|
|
// before destroying this builder.
|
|
// REQUIRES: Finish(), Abandon() have not been called
|
|
virtual void Abandon() = 0;
|
|
|
|
// Number of calls to Add() so far.
|
|
virtual uint64_t NumEntries() const = 0;
|
|
|
|
// Size of the file generated so far. If invoked after a successful
|
|
// Finish() call, returns the size of the final generated file.
|
|
virtual uint64_t FileSize() const = 0;
|
|
|
|
// If the user defined table properties collector suggest the file to
|
|
// be further compacted.
|
|
virtual bool NeedCompact() const { return false; }
|
|
|
|
// Returns table properties
|
|
virtual TableProperties GetTableProperties() const = 0;
|
|
};
|
|
|
|
} // namespace rocksdb
|