2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2013-12-05 22:09:13 +01:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <map>
|
|
|
|
#include <memory>
|
|
|
|
#include <string>
|
2015-09-02 22:58:22 +02:00
|
|
|
#include <vector>
|
2013-12-05 22:09:13 +01:00
|
|
|
|
2014-02-05 01:21:47 +01:00
|
|
|
#include "db/builder.h"
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
#include "db/table_properties_collector.h"
|
2015-10-16 23:10:33 +02:00
|
|
|
#include "util/kv_map.h"
|
2013-12-05 22:09:13 +01:00
|
|
|
#include "rocksdb/comparator.h"
|
|
|
|
#include "rocksdb/options.h"
|
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
#include "table/block_builder.h"
|
2014-07-19 01:58:13 +02:00
|
|
|
#include "table/format.h"
|
2013-12-05 22:09:13 +01:00
|
|
|
|
|
|
|
namespace rocksdb {
|
|
|
|
|
|
|
|
class BlockBuilder;
|
2013-12-06 01:51:26 +01:00
|
|
|
class BlockHandle;
|
|
|
|
class Env;
|
2014-05-01 20:09:32 +02:00
|
|
|
class Footer;
|
2013-12-05 22:09:13 +01:00
|
|
|
class Logger;
|
2013-12-06 01:51:26 +01:00
|
|
|
class RandomAccessFile;
|
2013-12-05 22:09:13 +01:00
|
|
|
struct TableProperties;
|
2015-10-13 00:06:38 +02:00
|
|
|
class InternalIterator;
|
2013-12-05 22:09:13 +01:00
|
|
|
|
|
|
|
class MetaIndexBuilder {
|
|
|
|
public:
|
|
|
|
MetaIndexBuilder(const MetaIndexBuilder&) = delete;
|
|
|
|
MetaIndexBuilder& operator=(const MetaIndexBuilder&) = delete;
|
|
|
|
|
|
|
|
MetaIndexBuilder();
|
|
|
|
void Add(const std::string& key, const BlockHandle& handle);
|
|
|
|
|
|
|
|
// Write all the added key/value pairs to the block and return the contents
|
|
|
|
// of the block.
|
|
|
|
Slice Finish();
|
|
|
|
|
|
|
|
private:
|
|
|
|
// store the sorted key/handle of the metablocks.
|
2015-09-02 22:58:22 +02:00
|
|
|
stl_wrappers::KVMap meta_block_handles_;
|
2013-12-05 22:09:13 +01:00
|
|
|
std::unique_ptr<BlockBuilder> meta_index_block_;
|
|
|
|
};
|
|
|
|
|
|
|
|
class PropertyBlockBuilder {
|
|
|
|
public:
|
|
|
|
PropertyBlockBuilder(const PropertyBlockBuilder&) = delete;
|
|
|
|
PropertyBlockBuilder& operator=(const PropertyBlockBuilder&) = delete;
|
|
|
|
|
|
|
|
PropertyBlockBuilder();
|
|
|
|
|
|
|
|
void AddTableProperty(const TableProperties& props);
|
|
|
|
void Add(const std::string& key, uint64_t value);
|
|
|
|
void Add(const std::string& key, const std::string& value);
|
|
|
|
void Add(const UserCollectedProperties& user_collected_properties);
|
|
|
|
|
|
|
|
// Write all the added entries to the block and return the block contents
|
|
|
|
Slice Finish();
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::unique_ptr<BlockBuilder> properties_block_;
|
2015-09-02 22:58:22 +02:00
|
|
|
stl_wrappers::KVMap props_;
|
2013-12-05 22:09:13 +01:00
|
|
|
};
|
|
|
|
|
|
|
|
// Were we encounter any error occurs during user-defined statistics collection,
|
|
|
|
// we'll write the warning message to info log.
|
|
|
|
void LogPropertiesCollectionError(
|
|
|
|
Logger* info_log, const std::string& method, const std::string& name);
|
|
|
|
|
|
|
|
// Utility functions help table builder to trigger batch events for user
|
|
|
|
// defined property collectors.
|
|
|
|
// Return value indicates if there is any error occurred; if error occurred,
|
|
|
|
// the warning message will be logged.
|
|
|
|
// NotifyCollectTableCollectorsOnAdd() triggers the `Add` event for all
|
|
|
|
// property collectors.
|
|
|
|
bool NotifyCollectTableCollectorsOnAdd(
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
const Slice& key, const Slice& value, uint64_t file_size,
|
|
|
|
const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors,
|
2013-12-05 22:09:13 +01:00
|
|
|
Logger* info_log);
|
|
|
|
|
|
|
|
// NotifyCollectTableCollectorsOnAdd() triggers the `Finish` event for all
|
|
|
|
// property collectors. The collected properties will be added to `builder`.
|
|
|
|
bool NotifyCollectTableCollectorsOnFinish(
|
A new call back to TablePropertiesCollector to allow users know the entry is add, delete or merge
Summary:
Currently users have no idea a key is add, delete or merge from TablePropertiesCollector call back. Add a new function to add it.
Also refactor the codes so that
(1) make table property collector and internal table property collector two separate data structures with the later one now exposed
(2) table builders only receive internal table properties
Test Plan: Add cases in table_properties_collector_test to cover both of old and new ways of using TablePropertiesCollector.
Reviewers: yhchiang, igor.sugak, rven, igor
Reviewed By: rven, igor
Subscribers: meyering, yoshinorim, maykov, leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D35373
2015-04-06 19:04:30 +02:00
|
|
|
const std::vector<std::unique_ptr<IntTblPropCollector>>& collectors,
|
TablePropertiesCollectorFactory
Summary:
This diff addresses task #4296714 and rethinks how users provide us with TablePropertiesCollectors as part of Options.
Here's description of task #4296714:
I'm debugging #4295529 and noticed that our count of user properties kDeletedKeys is wrong. We're sharing one single InternalKeyPropertiesCollector with all Table Builders. In LOG Files, we're outputting number of kDeletedKeys as connected with a single table, while it's actually the total count of deleted keys since creation of the DB.
For example, this table has 3155 entries and 1391828 deleted keys.
The problem with current approach that we call methods on a single TablePropertiesCollector for all the tables we create. Even worse, we could do it from multiple threads at the same time and TablePropertiesCollector has no way of knowing which table we're calling it for.
Good part: Looks like nobody inside Facebook is using Options::table_properties_collectors. This means we should be able to painfully change the API.
In this change, I introduce TablePropertiesCollectorFactory. For every table we create, we call `CreateTablePropertiesCollector`, which creates a TablePropertiesCollector for a single table. We then use it sequentially from a single thread, which means it doesn't have to be thread-safe.
Test Plan:
Added a test in table_properties_collector_test that fails on master (build two tables, assert that kDeletedKeys count is correct for the second one).
Also, all other tests
Reviewers: sdong, dhruba, haobo, kailiu
Reviewed By: kailiu
CC: leveldb
Differential Revision: https://reviews.facebook.net/D18579
2014-05-13 21:30:55 +02:00
|
|
|
Logger* info_log, PropertyBlockBuilder* builder);
|
2013-12-05 22:09:13 +01:00
|
|
|
|
2013-12-06 01:51:26 +01:00
|
|
|
// Read the properties from the table.
|
2014-02-08 04:26:49 +01:00
|
|
|
// @returns a status to indicate if the operation succeeded. On success,
|
|
|
|
// *table_properties will point to a heap-allocated TableProperties
|
|
|
|
// object, otherwise value of `table_properties` will not be modified.
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
|
|
|
|
const Footer& footer, Env* env, Logger* logger,
|
|
|
|
TableProperties** table_properties);
|
2013-12-06 01:51:26 +01:00
|
|
|
|
|
|
|
// Directly read the properties from the properties block of a plain table.
|
2014-02-08 04:26:49 +01:00
|
|
|
// @returns a status to indicate if the operation succeeded. On success,
|
|
|
|
// *table_properties will point to a heap-allocated TableProperties
|
|
|
|
// object, otherwise value of `table_properties` will not be modified.
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size,
|
2014-02-08 04:26:49 +01:00
|
|
|
uint64_t table_magic_number, Env* env,
|
|
|
|
Logger* info_log, TableProperties** properties);
|
2013-12-06 01:51:26 +01:00
|
|
|
|
2014-05-15 23:09:03 +02:00
|
|
|
// Find the meta block from the meta index block.
|
2015-10-13 00:06:38 +02:00
|
|
|
Status FindMetaBlock(InternalIterator* meta_index_iter,
|
2014-05-15 23:09:03 +02:00
|
|
|
const std::string& meta_block_name,
|
|
|
|
BlockHandle* block_handle);
|
|
|
|
|
2014-07-19 01:58:13 +02:00
|
|
|
// Find the meta block
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
|
2014-07-19 01:58:13 +02:00
|
|
|
uint64_t table_magic_number, Env* env,
|
|
|
|
const std::string& meta_block_name,
|
|
|
|
BlockHandle* block_handle);
|
|
|
|
|
|
|
|
// Read the specified meta block with name meta_block_name
|
|
|
|
// from `file` and initialize `contents` with contents of this block.
|
|
|
|
// Return Status::OK in case of success.
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
Status ReadMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
|
2014-07-19 01:58:13 +02:00
|
|
|
uint64_t table_magic_number, Env* env,
|
|
|
|
const std::string& meta_block_name,
|
|
|
|
BlockContents* contents);
|
|
|
|
|
2013-12-05 22:09:13 +01:00
|
|
|
} // namespace rocksdb
|