andreacavalli/netty5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Allow a limit to be set on the decompressed buffer size for ZlibDecoders (#9924)

Browse Source 
Motivation:
It is impossible to know in advance how much memory will be needed to
decompress a stream of bytes that was compressed using the DEFLATE
algorithm. In theory, up to 1032 times the compressed size could be
needed. For untrusted input, an attacker could exploit this to exhaust
the memory pool.

Modifications:
ZlibDecoder and its subclasses now support an optional limit on the size
of the decompressed buffer. By default, if the limit is reached,
decompression stops and a DecompressionException is thrown. Behavior
upon reaching the limit is modifiable by subclasses in case they desire
something else.

Result:
The decompressed buffer can now be limited to a configurable size, thus
mitigating the possibility of memory pool exhaustion.
This commit is contained in:
Rich DiCroce 2020-01-31 06:11:06 -05:00 committed by Norman Maurer
parent a6393c3d01
commit fa7e69bf30
8 changed files with 246 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
codec/src/main/java/io/netty/handler/codec/compression/JZlibDecoder.java
View File

@ -20,6 +20,7 @@ import static java.util.Objects.requireNonNull;
import com.jcraft.jzlib.Inflater;
import com.jcraft.jzlib.JZlib;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.ChannelHandlerContext;
public class JZlibDecoder extends ZlibDecoder {
@ -34,7 +35,21 @@ public class JZlibDecoder extends ZlibDecoder {
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder() {
this(ZlibWrapper.ZLIB);
this(ZlibWrapper.ZLIB, 0);
}
/**
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
* and specified maximum buffer allocation.
*
* @param maxAllocation
* Maximum size of the decompression buffer. Must be >= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(int maxAllocation) {
this(ZlibWrapper.ZLIB, maxAllocation);
}
/**
@ -43,6 +58,21 @@ public class JZlibDecoder extends ZlibDecoder {
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(ZlibWrapper wrapper) {
this(wrapper, 0);
}
/**
* Creates a new instance with the specified wrapper and maximum buffer allocation.
*
* @param maxAllocation
* Maximum size of the decompression buffer. Must be >= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
super(maxAllocation);
requireNonNull(wrapper, "wrapper");
int resultCode = z.init(ZlibUtil.convertWrapperType(wrapper));
@ -59,9 +89,23 @@ public class JZlibDecoder extends ZlibDecoder {
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(byte[] dictionary) {
requireNonNull(dictionary, "dictionary");
this.dictionary = dictionary;
this(dictionary, 0);
}
/**
* Creates a new instance with the specified preset dictionary and maximum buffer allocation.
* The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
* supports the preset dictionary.
*
* @param maxAllocation
* Maximum size of the decompression buffer. Must be >= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*
* @throws DecompressionException if failed to initialize zlib
*/
public JZlibDecoder(byte[] dictionary, int maxAllocation) {
super(maxAllocation);
this.dictionary = requireNonNull(dictionary, "dictionary");
int resultCode;
resultCode = z.inflateInit(JZlib.W_ZLIB);
if (resultCode != JZlib.Z_OK) {
@ -106,11 +150,11 @@ public class JZlibDecoder extends ZlibDecoder {
final int oldNextInIndex = z.next_in_index;
// Configure output.
ByteBuf decompressed = ctx.alloc().heapBuffer(inputLength << 1);
ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inputLength << 1);
try {
loop: for (;;) {
decompressed.ensureWritable(z.avail_in << 1);
decompressed = prepareDecompressBuffer(ctx, decompressed, z.avail_in << 1);
z.avail_out = decompressed.writableBytes();
z.next_out = decompressed.array();
z.next_out_index = decompressed.arrayOffset() + decompressed.writerIndex();
@ -166,4 +210,9 @@ public class JZlibDecoder extends ZlibDecoder {
z.next_out = null;
}
}
@Override
protected void decompressionBufferExhausted(ByteBuf buffer) {
finished = true;
}
}

71
codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java
View File

@ -18,6 +18,7 @@ package io.netty.handler.codec.compression;
import static java.util.Objects.requireNonNull;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.ChannelHandlerContext;
import java.util.zip.CRC32;
@ -65,7 +66,19 @@ public class JdkZlibDecoder extends ZlibDecoder {
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
*/
public JdkZlibDecoder() {
this(ZlibWrapper.ZLIB, null, false);
this(ZlibWrapper.ZLIB, null, false, 0);
}
/**
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB})
* and the specified maximum buffer allocation.
*
* @param maxAllocation
* Maximum size of the decompression buffer. Must be &gt;= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*/
public JdkZlibDecoder(int maxAllocation) {
this(ZlibWrapper.ZLIB, null, false, maxAllocation);
}
/**
@ -74,7 +87,20 @@ public class JdkZlibDecoder extends ZlibDecoder {
* supports the preset dictionary.
*/
public JdkZlibDecoder(byte[] dictionary) {
this(ZlibWrapper.ZLIB, dictionary, false);
this(ZlibWrapper.ZLIB, dictionary, false, 0);
}
/**
* Creates a new instance with the specified preset dictionary and maximum buffer allocation.
* The wrapper is always {@link ZlibWrapper#ZLIB} because it is the only format that
* supports the preset dictionary.
*
* @param maxAllocation
* Maximum size of the decompression buffer. Must be &gt;= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*/
public JdkZlibDecoder(byte[] dictionary, int maxAllocation) {
this(ZlibWrapper.ZLIB, dictionary, false, maxAllocation);
}
/**
@ -83,19 +109,43 @@ public class JdkZlibDecoder extends ZlibDecoder {
* supported atm.
*/
public JdkZlibDecoder(ZlibWrapper wrapper) {
this(wrapper, null, false);
this(wrapper, null, false, 0);
}
/**
* Creates a new instance with the specified wrapper and maximum buffer allocation.
* Be aware that only {@link ZlibWrapper#GZIP}, {@link ZlibWrapper#ZLIB} and {@link ZlibWrapper#NONE} are
* supported atm.
*
* @param maxAllocation
* Maximum size of the decompression buffer. Must be &gt;= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*/
public JdkZlibDecoder(ZlibWrapper wrapper, int maxAllocation) {
this(wrapper, null, false, maxAllocation);
}
public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated) {
this(wrapper, null, decompressConcatenated);
this(wrapper, null, decompressConcatenated, 0);
}
public JdkZlibDecoder(ZlibWrapper wrapper, boolean decompressConcatenated, int maxAllocation) {
this(wrapper, null, decompressConcatenated, maxAllocation);
}
public JdkZlibDecoder(boolean decompressConcatenated) {
this(ZlibWrapper.GZIP, null, decompressConcatenated);
this(ZlibWrapper.GZIP, null, decompressConcatenated, 0);
}
private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated) {
public JdkZlibDecoder(boolean decompressConcatenated, int maxAllocation) {
this(ZlibWrapper.GZIP, null, decompressConcatenated, maxAllocation);
}
private JdkZlibDecoder(ZlibWrapper wrapper, byte[] dictionary, boolean decompressConcatenated, int maxAllocation) {
super(maxAllocation);
requireNonNull(wrapper, "wrapper");
this.decompressConcatenated = decompressConcatenated;
switch (wrapper) {
case GZIP:
@ -176,7 +226,7 @@ public class JdkZlibDecoder extends ZlibDecoder {
inflater.setInput(array);
}
ByteBuf decompressed = ctx.alloc().heapBuffer(inflater.getRemaining() << 1);
ByteBuf decompressed = prepareDecompressBuffer(ctx, null, inflater.getRemaining() << 1);
try {
boolean readFooter = false;
while (!inflater.needsInput()) {
@ -207,7 +257,7 @@ public class JdkZlibDecoder extends ZlibDecoder {
}
break;
} else {
decompressed.ensureWritable(inflater.getRemaining() << 1);
decompressed = prepareDecompressBuffer(ctx, decompressed, inflater.getRemaining() << 1);
}
}
@ -237,6 +287,11 @@ public class JdkZlibDecoder extends ZlibDecoder {
}
}
@Override
protected void decompressionBufferExhausted(ByteBuf buffer) {
finished = true;
}
@Override
protected void handlerRemoved0(ChannelHandlerContext ctx) throws Exception {
super.handlerRemoved0(ctx);

65
codec/src/main/java/io/netty/handler/codec/compression/ZlibDecoder.java
View File

@ -16,6 +16,8 @@
package io.netty.handler.codec.compression;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ByteToMessageDecoder;
/**
@ -23,9 +25,72 @@ import io.netty.handler.codec.ByteToMessageDecoder;
*/
public abstract class ZlibDecoder extends ByteToMessageDecoder {
/**
* Maximum allowed size of the decompression buffer.
*/
protected final int maxAllocation;
/**
* Same as {@link #ZlibDecoder(int)} with maxAllocation = 0.
*/
public ZlibDecoder() {
this(0);
}
/**
* Construct a new ZlibDecoder.
* @param maxAllocation
* Maximum size of the decompression buffer. Must be &gt;= 0.
* If zero, maximum size is decided by the {@link ByteBufAllocator}.
*/
public ZlibDecoder(int maxAllocation) {
if (maxAllocation < 0) {
throw new IllegalArgumentException("maxAllocation must be >= 0");
}
this.maxAllocation = maxAllocation;
}
/**
* Returns {@code true} if and only if the end of the compressed stream
* has been reached.
*/
public abstract boolean isClosed();
/**
* Allocate or expand the decompression buffer, without exceeding the maximum allocation.
* Calls {@link #decompressionBufferExhausted(ByteBuf)} if the buffer is full and cannot be expanded further.
*/
protected ByteBuf prepareDecompressBuffer(ChannelHandlerContext ctx, ByteBuf buffer, int preferredSize) {
if (buffer == null) {
if (maxAllocation == 0) {
return ctx.alloc().heapBuffer(preferredSize);
}
return ctx.alloc().heapBuffer(Math.min(preferredSize, maxAllocation), maxAllocation);
}
// this always expands the buffer if possible, even if the expansion is less than preferredSize
// we throw the exception only if the buffer could not be expanded at all
// this means that one final attempt to deserialize will always be made with the buffer at maxAllocation
if (buffer.ensureWritable(preferredSize, true) == 1) {
// buffer must be consumed so subclasses don't add it to output
// we therefore duplicate it when calling decompressionBufferExhausted() to guarantee non-interference
// but wait until after to consume it so the subclass can tell how much output is really in the buffer
decompressionBufferExhausted(buffer.duplicate());
buffer.skipBytes(buffer.readableBytes());
throw new DecompressionException("Decompression buffer has reached maximum size: " + buffer.maxCapacity());
}
return buffer;
}
/**
* Called when the decompression buffer cannot be expanded further.
* Default implementation is a no-op, but subclasses can override in case they want to
* do something before the {@link DecompressionException} is thrown, such as log the
* data that was decompressed so far.
*/
protected void decompressionBufferExhausted(ByteBuf buffer) {
}
}

4
codec/src/test/java/io/netty/handler/codec/compression/JZlibTest.java
View File

@ -23,7 +23,7 @@ public class JZlibTest extends ZlibTest {
}
@Override
protected ZlibDecoder createDecoder(ZlibWrapper wrapper) {
return new JZlibDecoder(wrapper);
protected ZlibDecoder createDecoder(ZlibWrapper wrapper, int maxAllocation) {
return new JZlibDecoder(wrapper, maxAllocation);
}
}

4
codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java
View File

@ -38,8 +38,8 @@ public class JdkZlibTest extends ZlibTest {
}
@Override
protected ZlibDecoder createDecoder(ZlibWrapper wrapper) {
return new JdkZlibDecoder(wrapper);
protected ZlibDecoder createDecoder(ZlibWrapper wrapper, int maxAllocation) {
return new JdkZlibDecoder(wrapper, maxAllocation);
}
@Test(expected = DecompressionException.class)

4
codec/src/test/java/io/netty/handler/codec/compression/ZlibCrossTest1.java
View File

@ -23,7 +23,7 @@ public class ZlibCrossTest1 extends ZlibTest {
}
@Override
protected ZlibDecoder createDecoder(ZlibWrapper wrapper) {
return new JZlibDecoder(wrapper);
protected ZlibDecoder createDecoder(ZlibWrapper wrapper, int maxAllocation) {
return new JZlibDecoder(wrapper, maxAllocation);
}
}

4
codec/src/test/java/io/netty/handler/codec/compression/ZlibCrossTest2.java
View File

@ -25,8 +25,8 @@ public class ZlibCrossTest2 extends ZlibTest {
}
@Override
protected ZlibDecoder createDecoder(ZlibWrapper wrapper) {
return new JdkZlibDecoder(wrapper);
protected ZlibDecoder createDecoder(ZlibWrapper wrapper, int maxAllocation) {
return new JdkZlibDecoder(wrapper, maxAllocation);
}
@Test(expected = DecompressionException.class)

57
codec/src/test/java/io/netty/handler/codec/compression/ZlibTest.java
View File

@ -15,7 +15,9 @@
*/
package io.netty.handler.codec.compression;
import io.netty.buffer.AbstractByteBufAllocator;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.ByteBufInputStream;
import io.netty.buffer.Unpooled;
import io.netty.channel.embedded.EmbeddedChannel;
@ -88,8 +90,12 @@ public abstract class ZlibTest {
rand.nextBytes(BYTES_LARGE);
}
protected ZlibDecoder createDecoder(ZlibWrapper wrapper) {
return createDecoder(wrapper, 0);
}
protected abstract ZlibEncoder createEncoder(ZlibWrapper wrapper);
protected abstract ZlibDecoder createDecoder(ZlibWrapper wrapper);
protected abstract ZlibDecoder createDecoder(ZlibWrapper wrapper, int maxAllocation);
@Test
public void testGZIP2() throws Exception {
@ -345,6 +351,25 @@ public abstract class ZlibTest {
testCompressLarge(ZlibWrapper.GZIP, ZlibWrapper.ZLIB_OR_NONE);
}
@Test
public void testMaxAllocation() throws Exception {
int maxAllocation = 1024;
ZlibDecoder decoder = createDecoder(ZlibWrapper.ZLIB, maxAllocation);
EmbeddedChannel chDecoder = new EmbeddedChannel(decoder);
TestByteBufAllocator alloc = new TestByteBufAllocator(chDecoder.alloc());
chDecoder.config().setAllocator(alloc);
try {
chDecoder.writeInbound(Unpooled.wrappedBuffer(deflate(BYTES_LARGE)));
fail("decompressed size > maxAllocation, so should have thrown exception");
} catch (DecompressionException e) {
assertTrue(e.getMessage().startsWith("Decompression buffer has reached maximum size"));
assertEquals(maxAllocation, alloc.getMaxAllocation());
assertTrue(decoder.isClosed());
assertFalse(chDecoder.finish());
}
}
private static byte[] gzip(byte[] bytes) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream stream = new GZIPOutputStream(out);
@ -360,4 +385,34 @@ public abstract class ZlibTest {
stream.close();
return out.toByteArray();
}
private static final class TestByteBufAllocator extends AbstractByteBufAllocator {
private ByteBufAllocator wrapped;
private int maxAllocation;
TestByteBufAllocator(ByteBufAllocator wrapped) {
this.wrapped = wrapped;
}
public int getMaxAllocation() {
return maxAllocation;
}
@Override
public boolean isDirectBufferPooled() {
return wrapped.isDirectBufferPooled();
}
@Override
protected ByteBuf newHeapBuffer(int initialCapacity, int maxCapacity) {
maxAllocation = Math.max(maxAllocation, maxCapacity);
return wrapped.heapBuffer(initialCapacity, maxCapacity);
}
@Override
protected ByteBuf newDirectBuffer(int initialCapacity, int maxCapacity) {
maxAllocation = Math.max(maxAllocation, maxCapacity);
return wrapped.directBuffer(initialCapacity, maxCapacity);
}
}
}