From 3a41a97b0e00145ec20c699b8ec472cb42965756 Mon Sep 17 00:00:00 2001 From: Mads Johannessen <85683848+mjohannesse3@users.noreply.github.com> Date: Tue, 20 Jul 2021 15:21:26 +0200 Subject: [PATCH] Support large or variable chunk sizes (#11469) Motivation: Chunks are splitted up into even smaller chunks when the underlying buffer's readable bytes are less than the chunk size. The underlying buffer can be smaller than a chunk size if: - The chunk size is larger than the maximum plaintext chunk allowed by the TLS RFC, see: io.netty.handler.ssl.SslHandler.MAX_PLAINTEXT_LENGTH. - The chunk sizes are variable in size, which may cause Netty guess a buffer size that is smaller than a chunk size. Modification: Create a variable in HttpObjectDecoder: ByteBuf chunkedContent - Initialize chunkedContent in READ_CHUNK_SIZE with chunkSize as buffer size. - In READ_CHUNKED_CONTENT write bytes into chunkedContent - If the remaining chunk size is not 0 and toRead ==maxChunkSize, create a chunk using the chunkedContent and add it to the output messages before re-initializing chunkedContent with the remaining chunkSize as buffer size. - If the remaining chunk size is not 0 and toRead != maxChunkSize, return without adding any output messages. - If the remaining chunk size is 0, create a chunk using the chunkedContent and add it to the output messages; set chunkedContent = null and fall-through. Result: Support chunk sizes higher than the underlying buffer's readable bytes. Co-authored-by: Nitesh Kant Co-authored-by: Norman Maurer --- .../handler/codec/http/HttpClientCodec.java | 19 +++++-- .../handler/codec/http/HttpObjectDecoder.java | 29 ++++++++++ .../codec/http/HttpRequestDecoder.java | 31 +++++++++++ .../codec/http/HttpResponseDecoder.java | 31 +++++++++++ .../handler/codec/http/HttpServerCodec.java | 17 ++++++ .../codec/http/HttpResponseDecoderTest.java | 54 ++++++++++++++++++- 6 files changed, 177 insertions(+), 4 deletions(-) diff --git a/codec-http/src/main/java/io/netty/handler/codec/http/HttpClientCodec.java b/codec-http/src/main/java/io/netty/handler/codec/http/HttpClientCodec.java index bdc61b9708..9a66fbf833 100644 --- a/codec-http/src/main/java/io/netty/handler/codec/http/HttpClientCodec.java +++ b/codec-http/src/main/java/io/netty/handler/codec/http/HttpClientCodec.java @@ -29,6 +29,7 @@ import java.util.Queue; import java.util.concurrent.atomic.AtomicLong; import static io.netty.handler.codec.http.HttpObjectDecoder.DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS; +import static io.netty.handler.codec.http.HttpObjectDecoder.DEFAULT_ALLOW_PARTIAL_CHUNKS; import static io.netty.handler.codec.http.HttpObjectDecoder.DEFAULT_MAX_CHUNK_SIZE; import static io.netty.handler.codec.http.HttpObjectDecoder.DEFAULT_MAX_HEADER_SIZE; import static io.netty.handler.codec.http.HttpObjectDecoder.DEFAULT_MAX_INITIAL_LINE_LENGTH; @@ -136,8 +137,20 @@ public final class HttpClientCodec extends CombinedChannelDuplexHandlerRFC 7230, Section 3.3.2. * + * + * {@code allowPartialChunks} + * {@value #DEFAULT_ALLOW_PARTIAL_CHUNKS} + * If the length of a chunk exceeds the {@link ByteBuf}s readable bytes and {@code allowPartialChunks} + * is set to {@code true}, the chunk will be split into multiple {@link HttpContent}s. + * Otherwise, if the chunk size does not exceed {@code maxChunkSize} and {@code allowPartialChunks} + * is set to {@code false}, the {@link ByteBuf} is not decoded into an {@link HttpContent} until + * the readable bytes are greater or equal to the chunk size. + * * * *

Chunked Content

@@ -123,6 +132,7 @@ public abstract class HttpObjectDecoder extends ByteToMessageDecoder { public static final int DEFAULT_MAX_INITIAL_LINE_LENGTH = 4096; public static final int DEFAULT_MAX_HEADER_SIZE = 8192; public static final boolean DEFAULT_CHUNKED_SUPPORTED = true; + public static final boolean DEFAULT_ALLOW_PARTIAL_CHUNKS = true; public static final int DEFAULT_MAX_CHUNK_SIZE = 8192; public static final boolean DEFAULT_VALIDATE_HEADERS = true; public static final int DEFAULT_INITIAL_BUFFER_SIZE = 128; @@ -132,6 +142,7 @@ public abstract class HttpObjectDecoder extends ByteToMessageDecoder { private final int maxChunkSize; private final boolean chunkedSupported; + private final boolean allowPartialChunks; protected final boolean validateHeaders; private final boolean allowDuplicateContentLengths; private final HeaderParser headerParser; @@ -206,10 +217,24 @@ public abstract class HttpObjectDecoder extends ByteToMessageDecoder { DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS); } + /** + * Creates a new instance with the specified parameters. + */ protected HttpObjectDecoder( int maxInitialLineLength, int maxHeaderSize, int maxChunkSize, boolean chunkedSupported, boolean validateHeaders, int initialBufferSize, boolean allowDuplicateContentLengths) { + this(maxInitialLineLength, maxHeaderSize, maxChunkSize, chunkedSupported, validateHeaders, initialBufferSize, + allowDuplicateContentLengths, DEFAULT_ALLOW_PARTIAL_CHUNKS); + } + + /** + * Creates a new instance with the specified parameters. + */ + protected HttpObjectDecoder( + int maxInitialLineLength, int maxHeaderSize, int maxChunkSize, + boolean chunkedSupported, boolean validateHeaders, int initialBufferSize, + boolean allowDuplicateContentLengths, boolean allowPartialChunks) { checkPositive(maxInitialLineLength, "maxInitialLineLength"); checkPositive(maxHeaderSize, "maxHeaderSize"); checkPositive(maxChunkSize, "maxChunkSize"); @@ -221,6 +246,7 @@ public abstract class HttpObjectDecoder extends ByteToMessageDecoder { this.chunkedSupported = chunkedSupported; this.validateHeaders = validateHeaders; this.allowDuplicateContentLengths = allowDuplicateContentLengths; + this.allowPartialChunks = allowPartialChunks; } @Override @@ -366,6 +392,9 @@ public abstract class HttpObjectDecoder extends ByteToMessageDecoder { case READ_CHUNKED_CONTENT: { assert chunkSize <= Integer.MAX_VALUE; int toRead = Math.min((int) chunkSize, maxChunkSize); + if (!allowPartialChunks && buffer.readableBytes() < toRead) { + return; + } toRead = Math.min(toRead, buffer.readableBytes()); if (toRead == 0) { return; diff --git a/codec-http/src/main/java/io/netty/handler/codec/http/HttpRequestDecoder.java b/codec-http/src/main/java/io/netty/handler/codec/http/HttpRequestDecoder.java index d1d6d3df15..38e5f94205 100644 --- a/codec-http/src/main/java/io/netty/handler/codec/http/HttpRequestDecoder.java +++ b/codec-http/src/main/java/io/netty/handler/codec/http/HttpRequestDecoder.java @@ -51,6 +51,30 @@ import io.netty.handler.codec.TooLongFrameException; * after this decoder in the {@link ChannelPipeline}. * * + * + *

Parameters that control parsing behavior

+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
NameDefault valueMeaning
{@code allowDuplicateContentLengths}{@value #DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS}When set to {@code false}, will reject any messages that contain multiple Content-Length header fields. + * When set to {@code true}, will allow multiple Content-Length headers only if they are all the same decimal value. + * The duplicated field-values will be replaced with a single valid Content-Length field. + * See RFC 7230, Section 3.3.2.
{@code allowPartialChunks}{@value #DEFAULT_ALLOW_PARTIAL_CHUNKS}If the length of a chunk exceeds the {@link ByteBuf}s readable bytes and {@code allowPartialChunks} + * is set to {@code true}, the chunk will be split into multiple {@link HttpContent}s. + * Otherwise, if the chunk size does not exceed {@code maxChunkSize} and {@code allowPartialChunks} + * is set to {@code false}, the {@link ByteBuf} is not decoded into an {@link HttpContent} until + * the readable bytes are greater or equal to the chunk size.
*/ public class HttpRequestDecoder extends HttpObjectDecoder { @@ -89,6 +113,13 @@ public class HttpRequestDecoder extends HttpObjectDecoder { initialBufferSize, allowDuplicateContentLengths); } + public HttpRequestDecoder( + int maxInitialLineLength, int maxHeaderSize, int maxChunkSize, boolean validateHeaders, + int initialBufferSize, boolean allowDuplicateContentLengths, boolean allowPartialChunks) { + super(maxInitialLineLength, maxHeaderSize, maxChunkSize, DEFAULT_CHUNKED_SUPPORTED, validateHeaders, + initialBufferSize, allowDuplicateContentLengths, allowPartialChunks); + } + @Override protected HttpMessage createMessage(String[] initialLine) throws Exception { return new DefaultHttpRequest( diff --git a/codec-http/src/main/java/io/netty/handler/codec/http/HttpResponseDecoder.java b/codec-http/src/main/java/io/netty/handler/codec/http/HttpResponseDecoder.java index 97e38e10e7..37afa078f8 100644 --- a/codec-http/src/main/java/io/netty/handler/codec/http/HttpResponseDecoder.java +++ b/codec-http/src/main/java/io/netty/handler/codec/http/HttpResponseDecoder.java @@ -53,6 +53,30 @@ import io.netty.handler.codec.TooLongFrameException; * * * + *

Parameters that control parsing behavior

+ * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
NameDefault valueMeaning
{@code allowDuplicateContentLengths}{@value #DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS}When set to {@code false}, will reject any messages that contain multiple Content-Length header fields. + * When set to {@code true}, will allow multiple Content-Length headers only if they are all the same decimal value. + * The duplicated field-values will be replaced with a single valid Content-Length field. + * See RFC 7230, Section 3.3.2.
{@code allowPartialChunks}{@value #DEFAULT_ALLOW_PARTIAL_CHUNKS}If the length of a chunk exceeds the {@link ByteBuf}s readable bytes and {@code allowPartialChunks} + * is set to {@code true}, the chunk will be split into multiple {@link HttpContent}s. + * Otherwise, if the chunk size does not exceed {@code maxChunkSize} and {@code allowPartialChunks} + * is set to {@code false}, the {@link ByteBuf} is not decoded into an {@link HttpContent} until + * the readable bytes are greater or equal to the chunk size.
+ * *

Decoding a response for a HEAD request

*

* Unlike other HTTP requests, the successful response of a HEAD @@ -120,6 +144,13 @@ public class HttpResponseDecoder extends HttpObjectDecoder { initialBufferSize, allowDuplicateContentLengths); } + public HttpResponseDecoder( + int maxInitialLineLength, int maxHeaderSize, int maxChunkSize, boolean validateHeaders, + int initialBufferSize, boolean allowDuplicateContentLengths, boolean allowPartialChunks) { + super(maxInitialLineLength, maxHeaderSize, maxChunkSize, DEFAULT_CHUNKED_SUPPORTED, validateHeaders, + initialBufferSize, allowDuplicateContentLengths, allowPartialChunks); + } + @Override protected HttpMessage createMessage(String[] initialLine) { return new DefaultHttpResponse( diff --git a/codec-http/src/main/java/io/netty/handler/codec/http/HttpServerCodec.java b/codec-http/src/main/java/io/netty/handler/codec/http/HttpServerCodec.java index 0070029a9e..b8ffa01da9 100644 --- a/codec-http/src/main/java/io/netty/handler/codec/http/HttpServerCodec.java +++ b/codec-http/src/main/java/io/netty/handler/codec/http/HttpServerCodec.java @@ -85,6 +85,16 @@ public final class HttpServerCodec extends CombinedChannelDuplexHandler out) throws Exception { int oldSize = out.size(); diff --git a/codec-http/src/test/java/io/netty/handler/codec/http/HttpResponseDecoderTest.java b/codec-http/src/test/java/io/netty/handler/codec/http/HttpResponseDecoderTest.java index 0f2945994c..de93638925 100644 --- a/codec-http/src/test/java/io/netty/handler/codec/http/HttpResponseDecoderTest.java +++ b/codec-http/src/test/java/io/netty/handler/codec/http/HttpResponseDecoderTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; - +import java.util.Random; import static io.netty.handler.codec.http.HttpHeadersTestUtils.of; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; @@ -146,6 +146,58 @@ public class HttpResponseDecoderTest { assertNull(ch.readInbound()); } + @Test + public void testResponseDisallowPartialChunks() { + HttpResponseDecoder decoder = new HttpResponseDecoder( + HttpObjectDecoder.DEFAULT_MAX_INITIAL_LINE_LENGTH, + HttpObjectDecoder.DEFAULT_MAX_HEADER_SIZE, + HttpObjectDecoder.DEFAULT_MAX_CHUNK_SIZE, + HttpObjectDecoder.DEFAULT_VALIDATE_HEADERS, + HttpObjectDecoder.DEFAULT_INITIAL_BUFFER_SIZE, + HttpObjectDecoder.DEFAULT_ALLOW_DUPLICATE_CONTENT_LENGTHS, + false); + EmbeddedChannel ch = new EmbeddedChannel(decoder); + + String headers = "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n"; + assertTrue(ch.writeInbound(Unpooled.copiedBuffer(headers, CharsetUtil.US_ASCII))); + + HttpResponse res = ch.readInbound(); + assertThat(res.protocolVersion(), sameInstance(HttpVersion.HTTP_1_1)); + assertThat(res.status(), is(HttpResponseStatus.OK)); + + byte[] chunkBytes = new byte[10]; + Random random = new Random(); + random.nextBytes(chunkBytes); + final ByteBuf chunk = ch.alloc().buffer().writeBytes(chunkBytes); + final int chunkSize = chunk.readableBytes(); + ByteBuf partialChunk1 = chunk.retainedSlice(0, 5); + ByteBuf partialChunk2 = chunk.retainedSlice(5, 5); + + assertFalse(ch.writeInbound(Unpooled.copiedBuffer(Integer.toHexString(chunkSize) + + "\r\n", CharsetUtil.US_ASCII))); + assertFalse(ch.writeInbound(partialChunk1)); + assertTrue(ch.writeInbound(partialChunk2)); + + HttpContent content = ch.readInbound(); + assertEquals(chunk, content.content()); + content.release(); + chunk.release(); + + assertFalse(ch.writeInbound(Unpooled.copiedBuffer("\r\n", CharsetUtil.US_ASCII))); + + // Write the last chunk. + assertTrue(ch.writeInbound(Unpooled.copiedBuffer("0\r\n\r\n", CharsetUtil.US_ASCII))); + + // Ensure the last chunk was decoded. + HttpContent lastContent = ch.readInbound(); + assertFalse(lastContent.content().isReadable()); + lastContent.release(); + + assertFalse(ch.finish()); + } + @Test public void testResponseChunkedExceedMaxChunkSize() { EmbeddedChannel ch = new EmbeddedChannel(new HttpResponseDecoder(4096, 8192, 32));