rocksdb/thrift/lib/cpp/async/TNotificationPipe.h
Dhruba Borthakur 80c663882a Create leveldb server via Thrift.
Summary:
First draft.
Unit tests pass.

Test Plan: unit tests attached

Reviewers: heyongqiang

Reviewed By: heyongqiang

Differential Revision: https://reviews.facebook.net/D3969
2012-07-07 09:42:39 -07:00

338 lines
12 KiB
C++

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
#ifndef THRIFT_ASYNC_TNOTIFICATIONPIPE_H
#define THRIFT_ASYNC_TNOTIFICATIONPIPE_H 1
#include "thrift/lib/cpp/async/TDelayedDestruction.h"
#include "thrift/lib/cpp/async/TEventBase.h"
#include "thrift/lib/cpp/async/TEventHandler.h"
#include "thrift/lib/cpp/concurrency/Mutex.h"
#include <boost/shared_ptr.hpp>
#include <exception>
#include <limits.h>
namespace apache { namespace thrift { namespace async {
/**
* A simple notification pipe for sending messages to a TEventBase thread.
*
* TNotificationPipe is a unidirectional pipe for sending small, atomic
* messages.
*
* TNotificationPipe cannot be send messages larger than a fixed size.
* TNotificationPipe::kMaxMessageSize defines the maximum message size
* supported. If you need to pass larger amounts of data between threads,
* consider just passing a pointer to the data over the pipe, and using some
* external mechanism to synchronize management of the memory.
*
*
* TNotificationPipe provides two parallel APIs for writing and closing the
* pipe: a thread-safe version and a non-thread-safe version. Which version to
* use depends on how the caller uses the pipe:
*
* - If there is only a single writer thread, you can use the non-thread-safe
* versions of trySendMessage() and close(). This guarantees close() is
* never called by one thread while another thread is attempting to send a
* message.
*
* - If there are multiple writers, but the pipe is never closed by the
* writers, you can use the non-thread-safe version of trySendMessage().
* Multiple simultaneous trySendMessage() calls will not interfere with each
* other. Since none of the writer threads call close, a call to close()
* cannot be running simultaneously with a write attempt. (With this model,
* the TNotificationPipe is never closed until it is destroyed. It is up to
* the caller to ensure the TNotificationPipe is not destroyed while write
* threads still have a pointer or reference to it.)
*
* In other circumstances (if one thread may call close while another thread is
* simultaneously trying to write), the thread-safe versions
* trySendMessageSync() and closeSync() must be used.
*/
class TNotificationPipe : public TDelayedDestruction,
private TEventHandler,
private TEventBase::LoopCallback {
public:
/**
* A callback interface for receiving notification of messages from the pipe.
*/
class Callback {
public:
virtual ~Callback() {}
/**
* notificationMessage() will be invoked whenever a new
* message is available from the pipe.
*/
virtual void notificationMessage(const void *msg, uint32_t msgSize) = 0;
/**
* notificationPipeError() will be invoked if an error occurs while reading
* from the pipe. Before notificationPipeError() is invoked, the read
* callback will automatically be uninstalled and the pipe will be closed.
*/
virtual void notificationPipeError(const std::exception& ex) = 0;
/**
* notificationPipeClosed() is invoked in the read thread after the write
* end of the pipe is closed.
*/
virtual void notificationPipeClosed() = 0;
};
/**
* Helper function to create a new shared_ptr<TNotificationPipe>.
*
* This simply sets the correct destructor to call destroy() instead of
* directly deleting the TNotificationPipe.
*/
static boost::shared_ptr<TNotificationPipe> newPipe(TEventBase *base) {
return boost::shared_ptr<TNotificationPipe>(new TNotificationPipe(base),
Destructor());
}
/**
* Create a new TNotificationPipe.
*
* @param eventBase The TEventBase to use for receiving read notifications
* from this pipe. All read events will be processed in this
* TEventBase's thread. trySendMessage() may be called from any thread.
*/
TNotificationPipe(TEventBase *eventBase);
/**
* Destroy this TNotificationPipe.
*
* This method may only be called from the read thread.
*
* This will automatically close the pipe if it is not already closed.
*/
virtual void destroy();
/**
* Close the pipe.
*
* This version of close() is not thread-safe. It should only be used if the
* caller is sure no other thread is attempting to write a message at the
* same time.
*
* Use closeSync() if other threads may be attempting to send a message
* simultaneously. The other threads must use also use the thread-safe
* trySendMessageSync() or trySendFrameSync() calls.
*/
void close();
/**
* A thread-safe version of close().
*/
void closeSync();
/**
* Send a message over the pipe.
*
* trySendMessage() is best-effort. It will either immediately succeed to
* send the message, or it will fail immediately if the pipe reader is too
* busy and it's backlog of unread messages is too large.
*
* trySendMessage() also does not support arbitrarily large messages.
* It will also fail immediately if msgSize is larger than (PIPE_BUF - 4).
*
* If trySendMessage() succeeds, the message is guaranteed to be delivered to
* the pipe reader, except in the case where the pipe reader explicitly stops
* reading and destroys the pipe before processing all of its messages.
*
* On failure a TTransportException is thrown. The error code will be
* TTransportException::BAD_ARGS if the message is too large,
* TTransportException::TIMED_OUT if the message cannot be sent right now
* because the pipe is full, or TTransportException::NOT_OPEN if the pipe has
* already been closed.
*
* This method is thread safe with other simultaneous trySendMessage() calls,
* but not with close() calls. Use trySendMessageSync() and closeSync() if a
* close may occur simultaneously on another thread.
*/
void trySendMessage(const void *msg, uint32_t msgSize);
/**
* A thread-safe version of trySendMessage().
*
* This may be called simultaneously with closeSync().
*/
void trySendMessageSync(const void *msg, uint32_t msgSize);
/**
* Send a message over the pipe.
*
* This is identical to trySendMessage(), except that the caller must provide
* 4 bytes at the beginning of the message where we can write a frame length.
* This allows us to avoid copying the message into a new buffer.
* (trySendMessage() always has to make a copy of the message.)
*
* @param frame A pointer to the frame buffer. trySendFrame() will
* overwrite the first 4 bytes of this buffer. When the read callback
* receives the message, it will not see these first 4 bytes.
* @param frameSize The full size of the frame buffer. This must be at
* least 4 bytes long. The actual message size that will be sent is
* frameSize - 4.
*/
void trySendFrame(void *frame, uint32_t frameSize);
/**
* A thread-safe version of trySendFrame().
*
* This may be called simultaneously with closeSync().
*/
void trySendFrameSync(void *frame, uint32_t frameSize);
/**
* Get the number of messages which haven't been processed.
*/
int64_t getNumNotProcessed() const {
return numInputs_ - numOutputs_;
}
/**
* Set the callback to receive read notifications from this pipe.
*
* This method must be invoked from the pipe's read thread.
*
* May throw TLibraryException on error. The callback will always be unset
* (NULL) after an error.
*/
void setReadCallback(Callback *callback);
/**
* Mark the pipe read event handler as an "internal" event handler.
*
* This causes the notification pipe not to be counted when determining if
* the TEventBase has any more active events to wait on. This is intended to
* be used only be internal TEventBase code. This API is not guaranteed to
* remain stable or portable in the future.
*
* May throw TLibraryException if it fails to re-register its event handler
* with the correct flags.
*/
void setInternal(bool internal);
/**
* Get the maximum number of messages that will be read on a single iteration
* of the event loop.
*/
uint32_t getMaxReadAtOnce() const {
return maxReadAtOnce_;
}
/**
* Set the maximum number of messages to read each iteration of the event
* loop.
*
* If messages are being received faster than they can be processed, this
* helps limit the rate at which they will be read. This can be used to
* prevent the notification pipe reader from starving other users of the
* event loop.
*/
void setMaxReadAtOnce(uint32_t numMessages) {
maxReadAtOnce_ = numMessages;
}
/**
* The maximum message size that can be sent over a TNotificationPipe.
*
* This restriction ensures that trySendMessage() can send all messages
* atomically. This is (PIPE_BUF - 4) bytes. (On Linux, this is 4092
* bytes.)
*/
static const uint32_t kMaxMessageSize = PIPE_BUF - 4;
/**
* The default maximum number of messages that will be read each time around
* the event loop.
*
* This value used for each TNotificationPipe can be changed using the
* setMaxReadAtOnce() method.
*/
static const uint32_t kDefaultMaxReadAtOnce = 10;
private:
enum ReadAction {
kDoNothing,
kContinue,
kWaitForRead,
kRunInNextLoop,
};
// Forbidden copy constructor and assignment opererator
TNotificationPipe(TNotificationPipe const &);
TNotificationPipe& operator=(TNotificationPipe const &);
// TEventHandler methods
virtual void handlerReady(uint16_t events) THRIFT_NOEXCEPT;
// TEventBase::LoopCallback methods
virtual void runLoopCallback() THRIFT_NOEXCEPT;
void initPipe();
void registerPipeEvent();
void readMessages(ReadAction action);
ReadAction performRead();
ReadAction processReadData(uint32_t* messagesProcessed);
ReadAction handleError(const char* fmt, ...)
__attribute__((format(printf, 2, 3)));
void checkMessage(uint32_t msgSize);
void writeFrame(const void *frame, uint32_t frameSize);
TEventBase *eventBase_;
Callback *readCallback_;
int readPipe_;
int writePipe_;
bool internal_;
uint32_t maxReadAtOnce_;
int64_t numInputs_;
int64_t numOutputs_;
/**
* Mutex for guarding numInputs_
*/
concurrency::Mutex numInputsMutex_;
/**
* A mutex that guards writePipe_.
*
* This is used by closeSync(), trySendMessageSync(), and trySendFrameSync(),
* since trySendMessageSync() and trySendFrameSync() read writePipe_
* and closeSync() resets it to -1.
*/
concurrency::NoStarveReadWriteMutex writePipeMutex_;
/**
* A pointer to the end of valid read data in the read buffer.
*/
uint8_t *readPtr_;
/**
* An internal read buffer
*
* This is large enough to contain the maximum possible message plus the
* mssage length.
*/
uint8_t readBuffer_[kMaxMessageSize + 4];
};
}}} // apache::thrift::async
#endif // THRIFT_ASYNC_TNOTIFICATIONPIPE_H