2016-02-10 00:12:00 +01:00
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
2013-10-16 23:59:46 +02:00
|
|
|
// This source code is licensed under the BSD-style license found in the
|
|
|
|
// LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
// of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
//
|
2011-03-18 23:37:00 +01:00
|
|
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
2013-10-05 07:32:05 +02:00
|
|
|
#pragma once
|
2015-09-02 22:58:22 +02:00
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
#include <stdint.h>
|
2015-09-02 22:58:22 +02:00
|
|
|
|
|
|
|
#include <memory>
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
#include "db/log_format.h"
|
2013-08-23 17:38:13 +02:00
|
|
|
#include "rocksdb/slice.h"
|
|
|
|
#include "rocksdb/status.h"
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-10-04 06:49:15 +02:00
|
|
|
namespace rocksdb {
|
2011-03-18 23:37:00 +01:00
|
|
|
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
class WritableFileWriter;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
2013-01-20 11:07:13 +01:00
|
|
|
using std::unique_ptr;
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
namespace log {
|
|
|
|
|
2015-04-07 03:15:00 +02:00
|
|
|
/**
|
|
|
|
* Writer is a general purpose log stream writer. It provides an append-only
|
|
|
|
* abstraction for writing data. The details of the how the data is written is
|
|
|
|
* handled by the WriteableFile sub-class implementation.
|
|
|
|
*
|
|
|
|
* File format:
|
|
|
|
*
|
|
|
|
* File is broken down into variable sized records. The format of each record
|
|
|
|
* is described below.
|
|
|
|
* +-----+-------------+--+----+----------+------+-- ... ----+
|
|
|
|
* File | r0 | r1 |P | r2 | r3 | r4 | |
|
|
|
|
* +-----+-------------+--+----+----------+------+-- ... ----+
|
|
|
|
* <--- kBlockSize ------>|<-- kBlockSize ------>|
|
|
|
|
* rn = variable size records
|
|
|
|
* P = Padding
|
|
|
|
*
|
|
|
|
* Data is written out in kBlockSize chunks. If next record does not fit
|
|
|
|
* into the space left, the leftover space will be padded with \0.
|
|
|
|
*
|
2015-10-19 23:24:05 +02:00
|
|
|
* Legacy record format:
|
2015-04-07 03:15:00 +02:00
|
|
|
*
|
|
|
|
* +---------+-----------+-----------+--- ... ---+
|
|
|
|
* |CRC (4B) | Size (2B) | Type (1B) | Payload |
|
|
|
|
* +---------+-----------+-----------+--- ... ---+
|
|
|
|
*
|
|
|
|
* CRC = 32bit hash computed over the payload using CRC
|
|
|
|
* Size = Length of the payload data
|
|
|
|
* Type = Type of record
|
|
|
|
* (kZeroType, kFullType, kFirstType, kLastType, kMiddleType )
|
|
|
|
* The type is used to group a bunch of records together to represent
|
|
|
|
* blocks that are larger than kBlockSize
|
|
|
|
* Payload = Byte stream as long as specified by the payload size
|
|
|
|
*
|
2015-10-19 23:24:05 +02:00
|
|
|
* Recyclable record format:
|
|
|
|
*
|
|
|
|
* +---------+-----------+-----------+----------------+--- ... ---+
|
|
|
|
* |CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload |
|
|
|
|
* +---------+-----------+-----------+----------------+--- ... ---+
|
|
|
|
*
|
|
|
|
* Same as above, with the addition of
|
|
|
|
* Log number = 32bit log file number, so that we can distinguish between
|
|
|
|
* records written by the most recent log writer vs a previous one.
|
2015-04-07 03:15:00 +02:00
|
|
|
*/
|
2011-03-18 23:37:00 +01:00
|
|
|
class Writer {
|
|
|
|
public:
|
|
|
|
// Create a writer that will append data to "*dest".
|
|
|
|
// "*dest" must be initially empty.
|
|
|
|
// "*dest" must remain live while this Writer is in use.
|
2015-10-08 19:07:15 +02:00
|
|
|
explicit Writer(unique_ptr<WritableFileWriter>&& dest,
|
|
|
|
uint64_t log_number, bool recycle_log_files);
|
2011-03-18 23:37:00 +01:00
|
|
|
~Writer();
|
|
|
|
|
|
|
|
Status AddRecord(const Slice& slice);
|
|
|
|
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
WritableFileWriter* file() { return dest_.get(); }
|
|
|
|
const WritableFileWriter* file() const { return dest_.get(); }
|
2013-01-20 11:07:13 +01:00
|
|
|
|
2016-07-20 00:12:46 +02:00
|
|
|
uint64_t get_log_number() const { return log_number_; }
|
|
|
|
|
2011-03-18 23:37:00 +01:00
|
|
|
private:
|
Move rate_limiter, write buffering, most perf context instrumentation and most random kill out of Env
Summary: We want to keep Env a think layer for better portability. Less platform dependent codes should be moved out of Env. In this patch, I create a wrapper of file readers and writers, and put rate limiting, write buffering, as well as most perf context instrumentation and random kill out of Env. It will make it easier to maintain multiple Env in the future.
Test Plan: Run all existing unit tests.
Reviewers: anthony, kradhakrishnan, IslamAbdelRahman, yhchiang, igor
Reviewed By: igor
Subscribers: leveldb, dhruba
Differential Revision: https://reviews.facebook.net/D42321
2015-07-18 01:16:11 +02:00
|
|
|
unique_ptr<WritableFileWriter> dest_;
|
2015-11-15 19:49:14 +01:00
|
|
|
size_t block_offset_; // Current offset in block
|
2015-10-08 19:07:15 +02:00
|
|
|
uint64_t log_number_;
|
|
|
|
bool recycle_log_files_;
|
2011-03-18 23:37:00 +01:00
|
|
|
|
|
|
|
// crc32c values for all supported record types. These are
|
|
|
|
// pre-computed to reduce the overhead of computing the crc of the
|
|
|
|
// record type stored in the header.
|
|
|
|
uint32_t type_crc_[kMaxRecordType + 1];
|
|
|
|
|
|
|
|
Status EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
|
|
|
|
|
|
|
|
// No copying allowed
|
|
|
|
Writer(const Writer&);
|
|
|
|
void operator=(const Writer&);
|
|
|
|
};
|
|
|
|
|
2011-10-31 18:22:06 +01:00
|
|
|
} // namespace log
|
2013-10-04 06:49:15 +02:00
|
|
|
} // namespace rocksdb
|