Initial implementation of the Streaming API
This pull request provides a framework for exchanging a very large stream between handlers, typically between a decoder and an inbound handler (or between a handler that writes a message and an encoder that encodes that message). For example, an HTTP decoder, previously, generates multiple micro-messages to decode an HTTP message (i.e. HttpRequest + HttpChunks). With the streaming API, The HTTP decoder can simply generate a single HTTP message whose content is a Stream. And then the inbound handler can consume the Stream via the buffer you created when you begin to read the stream. If you create a buffer whose capacity is bounded, you can handle a very large stream without allocating a lot of memory. If you just want to wait until the whole content is ready, you can also do that with an unbounded buffer. The streaming API also supports a limited form of communication between a producer (i.e. decoder) and a consumer. A producer can abort the stream if the stream is not valid anymore. A consumer can choose to reject or discard the stream, where rejection is for unrecoverable failure and discard is for recoverable failure. P.S. Special thanks to @jpinner for the initial input.
This commit is contained in:
parent
b4bf565ad9
commit
32efba34d8
497
buffer/src/main/java/io/netty/buffer/stream/AbstractStream.java
Normal file
497
buffer/src/main/java/io/netty/buffer/stream/AbstractStream.java
Normal file
@ -0,0 +1,497 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.Buf;
|
||||||
|
import io.netty.util.concurrent.EventExecutor;
|
||||||
|
import io.netty.util.internal.PlatformDependent;
|
||||||
|
import io.netty.util.internal.logging.InternalLogger;
|
||||||
|
import io.netty.util.internal.logging.InternalLoggerFactory;
|
||||||
|
|
||||||
|
import java.util.Random;
|
||||||
|
|
||||||
|
public abstract class AbstractStream<T extends Buf> implements Stream<T> {
|
||||||
|
|
||||||
|
private static final InternalLogger logger = InternalLoggerFactory.getInstance(AbstractStream.class);
|
||||||
|
|
||||||
|
private static final Random random = new Random();
|
||||||
|
|
||||||
|
private static final ThreadLocal<Boolean> IN_CONSUMER = new ThreadLocal<Boolean>() {
|
||||||
|
@Override
|
||||||
|
protected Boolean initialValue() {
|
||||||
|
return Boolean.FALSE;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a random string that can be used for correlating a group of log messages.
|
||||||
|
*/
|
||||||
|
private static String nextLogKey() {
|
||||||
|
return Long.toHexString(random.nextInt() & 0xFFFFFFFFL);
|
||||||
|
}
|
||||||
|
|
||||||
|
private final StreamProducerContextImpl producerCtx;
|
||||||
|
private StreamConsumerContextImpl consumerCtx;
|
||||||
|
private Runnable invokeStreamConsumedTask;
|
||||||
|
|
||||||
|
/** 0 - init, 1 - accepted, 2 - discarded, 3 - rejected, 4 - closed */
|
||||||
|
int state;
|
||||||
|
T buffer;
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
protected AbstractStream(EventExecutor executor, StreamProducer<? super T> producer) {
|
||||||
|
producerCtx = new StreamProducerContextImpl(executor, producer);
|
||||||
|
}
|
||||||
|
|
||||||
|
T buffer() {
|
||||||
|
T buffer = this.buffer;
|
||||||
|
if (buffer == null) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void accept(EventExecutor executor, StreamConsumer<? super T> consumer) {
|
||||||
|
if (executor == null) {
|
||||||
|
throw new NullPointerException("executor");
|
||||||
|
}
|
||||||
|
if (consumer == null) {
|
||||||
|
throw new NullPointerException("handler");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state != 0) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
StreamConsumerContextImpl consumerCtx = new StreamConsumerContextImpl(executor, consumer);
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
StreamConsumer<T> h = (StreamConsumer<T>) consumer;
|
||||||
|
try {
|
||||||
|
buffer = h.newStreamBuffer(consumerCtx);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
PlatformDependent.throwException(t);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.consumerCtx = consumerCtx;
|
||||||
|
state = 1;
|
||||||
|
|
||||||
|
fireStreamAccepted();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamAccepted() {
|
||||||
|
EventExecutor e = producerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamAccepted();
|
||||||
|
} else {
|
||||||
|
e.execute(new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamAccepted();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamAccepted() {
|
||||||
|
StreamProducerContextImpl producerCtx = this.producerCtx;
|
||||||
|
try {
|
||||||
|
producerCtx.producer.streamAccepted(producerCtx);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
safeAbort(producerCtx, t);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (consumerCtx.nextCalled) {
|
||||||
|
consumerCtx.invokeStreamConsumed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void safeAbort(StreamProducerContextImpl producerCtx, Throwable cause) {
|
||||||
|
try {
|
||||||
|
producerCtx.abort(cause);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
if (logger.isWarnEnabled()) {
|
||||||
|
String key = nextLogKey();
|
||||||
|
logger.warn("[{}] Failed to auto-abort a stream.", key, t);
|
||||||
|
logger.warn("[{}] .. when invoked with the following cause:", key, cause);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void discard() {
|
||||||
|
StreamConsumerContextImpl consumerCtx = this.consumerCtx;
|
||||||
|
if (consumerCtx == null) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
consumerCtx.discard();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void reject(Throwable cause) {
|
||||||
|
StreamConsumerContextImpl consumerCtx = this.consumerCtx;
|
||||||
|
if (consumerCtx == null) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
consumerCtx.reject(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fail() {
|
||||||
|
switch (state) {
|
||||||
|
case 0:
|
||||||
|
throw new IllegalStateException("stream not accepted yet");
|
||||||
|
case 1:
|
||||||
|
throw new IllegalStateException("stream accepted already");
|
||||||
|
case 2:
|
||||||
|
throw new IllegalStateException("stream discarded already");
|
||||||
|
case 3:
|
||||||
|
throw new IllegalStateException("stream rejected already");
|
||||||
|
case 4:
|
||||||
|
throw new IllegalStateException("stream closed already");
|
||||||
|
default:
|
||||||
|
throw new Error();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class StreamProducerContextImpl implements StreamProducerContext<T> {
|
||||||
|
|
||||||
|
final EventExecutor executor;
|
||||||
|
final StreamProducer<T> producer;
|
||||||
|
boolean invokedStreamOpen;
|
||||||
|
Runnable invokeStreamUpdatedTask;
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
StreamProducerContextImpl(EventExecutor executor, StreamProducer<? super T> producer) {
|
||||||
|
if (executor == null) {
|
||||||
|
throw new NullPointerException("executor");
|
||||||
|
}
|
||||||
|
if (producer == null) {
|
||||||
|
throw new NullPointerException("producer");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.executor = executor;
|
||||||
|
this.producer = (StreamProducer<T>) producer;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventExecutor executor() {
|
||||||
|
return executor;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public T buffer() {
|
||||||
|
return AbstractStream.this.buffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public StreamProducerContext<T> update() {
|
||||||
|
if (state != 1) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
fireStreamUpdated();
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamUpdated() {
|
||||||
|
EventExecutor e = consumerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamUpdated();
|
||||||
|
} else {
|
||||||
|
Runnable task = invokeStreamUpdatedTask;
|
||||||
|
if (task == null) {
|
||||||
|
invokeStreamUpdatedTask = task = new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamUpdated();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
e.execute(task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamUpdated() {
|
||||||
|
StreamConsumerContextImpl consumerCtx = AbstractStream.this.consumerCtx;
|
||||||
|
StreamConsumer<T> consumer = consumerCtx.consumer;
|
||||||
|
if (consumerCtx.singleThreaded) {
|
||||||
|
IN_CONSUMER.set(Boolean.TRUE);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
if (!invokedStreamOpen) {
|
||||||
|
invokedStreamOpen = true;
|
||||||
|
try {
|
||||||
|
consumer.streamOpen(consumerCtx);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
safeAbort(producerCtx, new StreamConsumerException(t));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
consumer.streamUpdated(consumerCtx);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
safeAbort(producerCtx, new StreamConsumerException(t));
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (consumerCtx.singleThreaded) {
|
||||||
|
IN_CONSUMER.set(Boolean.FALSE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
if (state == 1) {
|
||||||
|
state = 4;
|
||||||
|
fireStreamClosed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamClosed() {
|
||||||
|
EventExecutor e = consumerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamClosed();
|
||||||
|
} else {
|
||||||
|
e.execute(new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamClosed();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamClosed() {
|
||||||
|
StreamConsumerContextImpl consumerCtx = AbstractStream.this.consumerCtx;
|
||||||
|
try {
|
||||||
|
consumerCtx.consumer.streamClosed(consumerCtx);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("StreamConsumer.streamClosed() raised an exception.", t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void abort(Throwable cause) {
|
||||||
|
if (cause == null) {
|
||||||
|
throw new NullPointerException("cause");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state != 1) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
fireStreamAborted(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamAborted(final Throwable cause) {
|
||||||
|
StreamConsumerContextImpl consumerCtx = AbstractStream.this.consumerCtx;
|
||||||
|
EventExecutor e = consumerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamAborted(cause);
|
||||||
|
} else {
|
||||||
|
e.execute(new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamAborted(cause);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamAborted(Throwable cause) {
|
||||||
|
StreamConsumerContextImpl consumerCtx = AbstractStream.this.consumerCtx;
|
||||||
|
try {
|
||||||
|
consumerCtx.consumer.streamAborted(consumerCtx, cause);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
if (logger.isWarnEnabled()) {
|
||||||
|
String key = nextLogKey();
|
||||||
|
logger.warn("[{}] StreamConsumer.streamAborted() raised an exception.", key, t);
|
||||||
|
logger.warn("[{}] .. when invoked with the following cause:", key, cause);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
invokeStreamClosed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private final class StreamConsumerContextImpl implements StreamConsumerContext<T> {
|
||||||
|
|
||||||
|
final EventExecutor executor;
|
||||||
|
final StreamConsumer<T> consumer;
|
||||||
|
final boolean singleThreaded;
|
||||||
|
boolean nextCalled;
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
StreamConsumerContextImpl(EventExecutor executor, StreamConsumer<? super T> consumer) {
|
||||||
|
if (executor == null) {
|
||||||
|
throw new NullPointerException("executor");
|
||||||
|
}
|
||||||
|
if (consumer == null) {
|
||||||
|
throw new NullPointerException("consumer");
|
||||||
|
}
|
||||||
|
this.executor = executor;
|
||||||
|
this.consumer = (StreamConsumer<T>) consumer;
|
||||||
|
singleThreaded = executor == producerCtx.executor;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public EventExecutor executor() {
|
||||||
|
return executor;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public T buffer() {
|
||||||
|
return AbstractStream.this.buffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void discard() {
|
||||||
|
switch (state) {
|
||||||
|
case 2:
|
||||||
|
return;
|
||||||
|
case 3:
|
||||||
|
case 4:
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
T buffer = AbstractStream.this.buffer;
|
||||||
|
if (buffer != null) {
|
||||||
|
buffer.release();
|
||||||
|
}
|
||||||
|
|
||||||
|
state = 2;
|
||||||
|
|
||||||
|
fireStreamDiscarded();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamDiscarded() {
|
||||||
|
EventExecutor e = producerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamDiscarded();
|
||||||
|
} else {
|
||||||
|
e.execute(new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamDiscarded();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamDiscarded() {
|
||||||
|
StreamProducerContextImpl producerCtx = AbstractStream.this.producerCtx;
|
||||||
|
try {
|
||||||
|
producerCtx.producer.streamDiscarded(producerCtx);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
logger.warn("StreamProducer.streamDiscarded() raised an exception.", t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void reject(Throwable cause) {
|
||||||
|
if (cause == null) {
|
||||||
|
throw new NullPointerException("cause");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state != 1) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
T buffer = AbstractStream.this.buffer;
|
||||||
|
if (buffer != null) {
|
||||||
|
buffer.release();
|
||||||
|
}
|
||||||
|
|
||||||
|
state = 3;
|
||||||
|
|
||||||
|
fireStreamRejected(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamRejected(final Throwable cause) {
|
||||||
|
EventExecutor e = producerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamRejected(cause);
|
||||||
|
} else {
|
||||||
|
e.execute(new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamRejected(cause);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamRejected(Throwable cause) {
|
||||||
|
StreamProducerContextImpl producerCtx = AbstractStream.this.producerCtx;
|
||||||
|
try {
|
||||||
|
producerCtx.producer.streamRejected(producerCtx, cause);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
if (logger.isWarnEnabled()) {
|
||||||
|
String key = nextLogKey();
|
||||||
|
logger.warn("[{}] StreamProducer.streamRejected() raised an exception.", key, t);
|
||||||
|
logger.warn("[{}] .. when invoked with the following cause:", key, cause);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void next() {
|
||||||
|
if (state != 1) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (singleThreaded && IN_CONSUMER.get()) {
|
||||||
|
nextCalled = true;
|
||||||
|
} else {
|
||||||
|
fireStreamConsumed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void fireStreamConsumed() {
|
||||||
|
EventExecutor e = producerCtx.executor;
|
||||||
|
if (e.inEventLoop()) {
|
||||||
|
invokeStreamConsumed();
|
||||||
|
} else {
|
||||||
|
Runnable task = invokeStreamConsumedTask;
|
||||||
|
if (task == null) {
|
||||||
|
invokeStreamConsumedTask = task = new Runnable() {
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
invokeStreamConsumed();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
e.execute(task);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void invokeStreamConsumed() {
|
||||||
|
StreamProducerContextImpl producerCtx = AbstractStream.this.producerCtx;
|
||||||
|
try {
|
||||||
|
do {
|
||||||
|
consumerCtx.nextCalled = false;
|
||||||
|
producerCtx.producer.streamConsumed(producerCtx);
|
||||||
|
} while (consumerCtx.nextCalled);
|
||||||
|
} catch (Throwable t) {
|
||||||
|
reject(new StreamProducerException(t));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
26
buffer/src/main/java/io/netty/buffer/stream/Stream.java
Normal file
26
buffer/src/main/java/io/netty/buffer/stream/Stream.java
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.Buf;
|
||||||
|
import io.netty.util.concurrent.EventExecutor;
|
||||||
|
|
||||||
|
public interface Stream<T extends Buf> {
|
||||||
|
void accept(EventExecutor executor, StreamConsumer<? super T> consumer);
|
||||||
|
void discard();
|
||||||
|
void reject(Throwable cause);
|
||||||
|
}
|
@ -0,0 +1,28 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.Buf;
|
||||||
|
|
||||||
|
public interface StreamConsumer<T extends Buf> {
|
||||||
|
T newStreamBuffer(StreamConsumerContext<T> ctx) throws Exception;
|
||||||
|
|
||||||
|
void streamOpen(StreamConsumerContext<T> ctx) throws Exception;
|
||||||
|
void streamUpdated(StreamConsumerContext<T> ctx) throws Exception;
|
||||||
|
void streamAborted(StreamConsumerContext<T> ctx, Throwable cause) throws Exception;
|
||||||
|
void streamClosed(StreamConsumerContext<T> ctx) throws Exception;
|
||||||
|
}
|
@ -0,0 +1,28 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.Buf;
|
||||||
|
import io.netty.util.concurrent.EventExecutor;
|
||||||
|
|
||||||
|
public interface StreamConsumerContext<T extends Buf> {
|
||||||
|
EventExecutor executor();
|
||||||
|
T buffer();
|
||||||
|
void next();
|
||||||
|
void discard();
|
||||||
|
void reject(Throwable cause);
|
||||||
|
}
|
@ -0,0 +1,36 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
public class StreamConsumerException extends StreamException {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = -7688455132621735741L;
|
||||||
|
|
||||||
|
public StreamConsumerException() { }
|
||||||
|
|
||||||
|
public StreamConsumerException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StreamConsumerException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StreamConsumerException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,36 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
public class StreamException extends RuntimeException {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = -8529582560484984135L;
|
||||||
|
|
||||||
|
public StreamException() { }
|
||||||
|
|
||||||
|
public StreamException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StreamException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StreamException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,26 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.Buf;
|
||||||
|
|
||||||
|
public interface StreamProducer<T extends Buf> {
|
||||||
|
void streamAccepted(StreamProducerContext<T> ctx) throws Exception;
|
||||||
|
void streamConsumed(StreamProducerContext<T> ctx) throws Exception;
|
||||||
|
void streamDiscarded(StreamProducerContext<T> ctx) throws Exception;
|
||||||
|
void streamRejected(StreamProducerContext<T> ctx, Throwable cause) throws Exception;
|
||||||
|
}
|
@ -0,0 +1,28 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.Buf;
|
||||||
|
import io.netty.util.concurrent.EventExecutor;
|
||||||
|
|
||||||
|
public interface StreamProducerContext<T extends Buf> {
|
||||||
|
EventExecutor executor();
|
||||||
|
T buffer();
|
||||||
|
StreamProducerContext<T> update();
|
||||||
|
void close();
|
||||||
|
void abort(Throwable cause);
|
||||||
|
}
|
@ -0,0 +1,36 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
public class StreamProducerException extends StreamException {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 8830744325775707415L;
|
||||||
|
|
||||||
|
public StreamProducerException() { }
|
||||||
|
|
||||||
|
public StreamProducerException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StreamProducerException(String message, Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public StreamProducerException(Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
}
|
174
buffer/src/test/java/io/netty/buffer/stream/StreamTest.java
Normal file
174
buffer/src/test/java/io/netty/buffer/stream/StreamTest.java
Normal file
@ -0,0 +1,174 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2013 The Netty Project
|
||||||
|
*
|
||||||
|
* The Netty Project licenses this file to you under the Apache License,
|
||||||
|
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at:
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.netty.buffer.stream;
|
||||||
|
|
||||||
|
import io.netty.buffer.ByteBuf;
|
||||||
|
import io.netty.buffer.Unpooled;
|
||||||
|
import io.netty.util.concurrent.DefaultEventExecutorGroup;
|
||||||
|
import io.netty.util.concurrent.DefaultPromise;
|
||||||
|
import io.netty.util.concurrent.EventExecutor;
|
||||||
|
import io.netty.util.concurrent.Promise;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class StreamTest {
|
||||||
|
|
||||||
|
private static EventExecutor executorA;
|
||||||
|
private static EventExecutor executorB;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void setUp() {
|
||||||
|
executorA = new DefaultEventExecutorGroup(1).next();
|
||||||
|
executorB = new DefaultEventExecutorGroup(1).next();
|
||||||
|
}
|
||||||
|
|
||||||
|
@AfterClass
|
||||||
|
public static void tearDown() {
|
||||||
|
executorB.shutdown();
|
||||||
|
executorA.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 5000)
|
||||||
|
public void testSimpleWithSameExecutor() throws Exception {
|
||||||
|
final long size = 1048576L * 1024L * 1024L; // Transfer whooping 1 TiB of garbage
|
||||||
|
testSimple0(executorA, executorA, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testSimpleWithDifferentExecutors() throws Exception {
|
||||||
|
final long size = 1048576L * 1024L * 32L; // Transfer 32 GiB of garbage
|
||||||
|
testSimple0(executorA, executorB, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void testSimple0(EventExecutor executorA, EventExecutor executorB, long size) throws Exception {
|
||||||
|
LargeByteStreamConsumer consumer = new LargeByteStreamConsumer(size);
|
||||||
|
Stream<ByteBuf> stream = new LargeByteStream(executorA, size);
|
||||||
|
stream.accept(executorB, consumer);
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (consumer.future.await(1000)) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
System.err.println(consumer.counter + " / " + size);
|
||||||
|
}
|
||||||
|
|
||||||
|
consumer.future.sync();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class LargeByteStream extends AbstractStream<ByteBuf> {
|
||||||
|
LargeByteStream(EventExecutor executor, long size) {
|
||||||
|
super(executor, new LargeByteStreamProducer(size));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class LargeByteStreamProducer implements StreamProducer<ByteBuf> {
|
||||||
|
|
||||||
|
private final long size;
|
||||||
|
private long counter;
|
||||||
|
|
||||||
|
LargeByteStreamProducer(long size) {
|
||||||
|
this.size = size;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamAccepted(StreamProducerContext<ByteBuf> ctx) throws Exception {
|
||||||
|
generate(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamConsumed(StreamProducerContext<ByteBuf> ctx) throws Exception {
|
||||||
|
generate(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void generate(StreamProducerContext<ByteBuf> ctx) {
|
||||||
|
ByteBuf buf = ctx.buffer();
|
||||||
|
int chunkSize = (int) Math.min(size - counter, buf.maxWritableBytes());
|
||||||
|
buf.ensureWritable(chunkSize);
|
||||||
|
buf.writerIndex(buf.writerIndex() + chunkSize);
|
||||||
|
ctx.update();
|
||||||
|
counter += chunkSize;
|
||||||
|
if (counter == size) {
|
||||||
|
ctx.close();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (counter > size) {
|
||||||
|
throw new AssertionError("counter > size");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamDiscarded(StreamProducerContext<ByteBuf> ctx) throws Exception {
|
||||||
|
throw new AssertionError("stream discarded");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamRejected(StreamProducerContext<ByteBuf> ctx, Throwable cause) throws Exception {
|
||||||
|
throw new AssertionError("stream rejected", cause);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class LargeByteStreamConsumer implements StreamConsumer<ByteBuf> {
|
||||||
|
|
||||||
|
private final long size;
|
||||||
|
volatile Promise future;
|
||||||
|
volatile long counter;
|
||||||
|
|
||||||
|
LargeByteStreamConsumer(long size) {
|
||||||
|
this.size = size;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ByteBuf newStreamBuffer(StreamConsumerContext<ByteBuf> ctx) throws Exception {
|
||||||
|
future = new DefaultPromise(ctx.executor());
|
||||||
|
return Unpooled.buffer(0, 65536); // Only use 64 KiB at max.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamOpen(StreamConsumerContext<ByteBuf> ctx) throws Exception { }
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamUpdated(StreamConsumerContext<ByteBuf> ctx) throws Exception {
|
||||||
|
ByteBuf buf = ctx.buffer();
|
||||||
|
int chunkSize = buf.readableBytes();
|
||||||
|
buf.skipBytes(chunkSize);
|
||||||
|
buf.discardReadBytes();
|
||||||
|
counter += chunkSize;
|
||||||
|
if (counter == size) {
|
||||||
|
future.setSuccess();
|
||||||
|
} else if (counter > size) {
|
||||||
|
AssertionError error = new AssertionError("counter > size");
|
||||||
|
ctx.reject(error);
|
||||||
|
future.setFailure(error);
|
||||||
|
} else {
|
||||||
|
ctx.next();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamAborted(StreamConsumerContext<ByteBuf> ctx, Throwable cause) throws Exception {
|
||||||
|
future.setFailure(cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void streamClosed(StreamConsumerContext<ByteBuf> ctx) throws Exception {
|
||||||
|
future.tryFailure(new AssertionError("tryFailure() must return false."));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user