b45b1cde3e
Summary: This is a feature to sample data-block compressibility and and report them as stats. 1 in N (tunable) blocks is sampled for compressibility using two algorithms: 1. lz4 or snappy for fast compression 2. zstd or zlib for slow but higher compression. The stats are reported to the caller as raw-bytes and compressed-bytes. The block continues to be compressed for storage using the specified CompressionType. The db_bench_tool how has a command line option for specifying the sampling rate. It's default value is 0 (no sampling). To test the overhead for a certain value, users can compare the performance of db_bench_tool, varying the sampling rate. It is unlikely to have a noticeable impact for high values like 20. Pull Request resolved: https://github.com/facebook/rocksdb/pull/4842 Differential Revision: D13629011 Pulled By: shobhitdayal fbshipit-source-id: 14ca668bcab6499b2a1734edf848eb62a4f4fafa
108 lines
3.5 KiB
C++
108 lines
3.5 KiB
C++
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// This file defines a collection of statistics collectors.
|
|
#pragma once
|
|
|
|
#include "rocksdb/table_properties.h"
|
|
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
namespace rocksdb {
|
|
|
|
// Base class for internal table properties collector.
|
|
class IntTblPropCollector {
|
|
public:
|
|
virtual ~IntTblPropCollector() {}
|
|
virtual Status Finish(UserCollectedProperties* properties) = 0;
|
|
|
|
virtual const char* Name() const = 0;
|
|
|
|
// @params key the user key that is inserted into the table.
|
|
// @params value the value that is inserted into the table.
|
|
virtual Status InternalAdd(const Slice& key, const Slice& value,
|
|
uint64_t file_size) = 0;
|
|
|
|
virtual void BlockAdd(uint64_t blockRawBytes,
|
|
uint64_t blockCompressedBytesFast,
|
|
uint64_t blockCompressedBytesSlow) = 0;
|
|
|
|
virtual UserCollectedProperties GetReadableProperties() const = 0;
|
|
|
|
virtual bool NeedCompact() const { return false; }
|
|
};
|
|
|
|
// Factory for internal table properties collector.
|
|
class IntTblPropCollectorFactory {
|
|
public:
|
|
virtual ~IntTblPropCollectorFactory() {}
|
|
// has to be thread-safe
|
|
virtual IntTblPropCollector* CreateIntTblPropCollector(
|
|
uint32_t column_family_id) = 0;
|
|
|
|
// The name of the properties collector can be used for debugging purpose.
|
|
virtual const char* Name() const = 0;
|
|
};
|
|
|
|
// When rocksdb creates a new table, it will encode all "user keys" into
|
|
// "internal keys", which contains meta information of a given entry.
|
|
//
|
|
// This class extracts user key from the encoded internal key when Add() is
|
|
// invoked.
|
|
class UserKeyTablePropertiesCollector : public IntTblPropCollector {
|
|
public:
|
|
// transfer of ownership
|
|
explicit UserKeyTablePropertiesCollector(TablePropertiesCollector* collector)
|
|
: collector_(collector) {}
|
|
|
|
virtual ~UserKeyTablePropertiesCollector() {}
|
|
|
|
virtual Status InternalAdd(const Slice& key, const Slice& value,
|
|
uint64_t file_size) override;
|
|
|
|
virtual void BlockAdd(uint64_t blockRawBytes,
|
|
uint64_t blockCompressedBytesFast,
|
|
uint64_t blockCompressedBytesSlow) override;
|
|
|
|
virtual Status Finish(UserCollectedProperties* properties) override;
|
|
|
|
virtual const char* Name() const override { return collector_->Name(); }
|
|
|
|
UserCollectedProperties GetReadableProperties() const override;
|
|
|
|
virtual bool NeedCompact() const override {
|
|
return collector_->NeedCompact();
|
|
}
|
|
|
|
protected:
|
|
std::unique_ptr<TablePropertiesCollector> collector_;
|
|
};
|
|
|
|
class UserKeyTablePropertiesCollectorFactory
|
|
: public IntTblPropCollectorFactory {
|
|
public:
|
|
explicit UserKeyTablePropertiesCollectorFactory(
|
|
std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory)
|
|
: user_collector_factory_(user_collector_factory) {}
|
|
virtual IntTblPropCollector* CreateIntTblPropCollector(
|
|
uint32_t column_family_id) override {
|
|
TablePropertiesCollectorFactory::Context context;
|
|
context.column_family_id = column_family_id;
|
|
return new UserKeyTablePropertiesCollector(
|
|
user_collector_factory_->CreateTablePropertiesCollector(context));
|
|
}
|
|
|
|
virtual const char* Name() const override {
|
|
return user_collector_factory_->Name();
|
|
}
|
|
|
|
private:
|
|
std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory_;
|
|
};
|
|
|
|
} // namespace rocksdb
|