From 24baa9a2ac2d04c602c4eda72fd8edb04fc922cd Mon Sep 17 00:00:00 2001 From: Jeff Pinner Date: Wed, 1 Feb 2012 16:00:10 -0800 Subject: [PATCH] Fix #163: HttpContentCompressor consumes too much memory --- .../codec/compression/ZlibEncoder.java | 105 +++++++++++++++--- .../codec/http/HttpContentCompressor.java | 47 +++++++- .../netty/util/internal/jzlib/Deflate.java | 4 +- .../netty/util/internal/jzlib/ZStream.java | 8 +- 4 files changed, 142 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java b/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java index 5925d6fb89..bd518755ef 100644 --- a/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java +++ b/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java @@ -46,7 +46,8 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne private volatile ChannelHandlerContext ctx; /** - * Creates a new zlib encoder with the default compression level ({@code 6}) + * Creates a new zlib encoder with the default compression level ({@code 6}), + * default window bits ({@code 15}), default memory level ({@code 8}), * and the default wrapper ({@link ZlibWrapper#ZLIB}). * * @throws CompressionException if failed to initialize zlib @@ -56,7 +57,8 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne } /** - * Creates a new zlib encoder with the specified {@code compressionLevel} + * Creates a new zlib encoder with the specified {@code compressionLevel}, + * default window bits ({@code 15}), default memory level ({@code 8}), * and the default wrapper ({@link ZlibWrapper#ZLIB}). * * @param compressionLevel @@ -71,7 +73,8 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne } /** - * Creates a new zlib encoder with the default compression level ({@code 6}) + * Creates a new zlib encoder with the default compression level ({@code 6}), + * default window bits ({@code 15}), default memory level ({@code 8}), * and the specified wrapper. * * @throws CompressionException if failed to initialize zlib @@ -81,8 +84,10 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne } /** - * Creates a new zlib encoder with the specified {@code compressionLevel} + * Creates a new zlib encoder with the specified {@code compressionLevel}, + * default window bits ({@code 15}), default memory level ({@code 8}), * and the specified wrapper. + * * @param compressionLevel * {@code 1} yields the fastest compression and {@code 9} yields the * best compression. {@code 0} means no compression. The default @@ -91,10 +96,43 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne * @throws CompressionException if failed to initialize zlib */ public ZlibEncoder(ZlibWrapper wrapper, int compressionLevel) { + this(wrapper, compressionLevel, 15, 8); + } + + /** + * Creates a new zlib encoder with the specified {@code compressionLevel}, + * the specified {@code windowBits}, the specified {@code memLevel}, and + * the specified wrapper. + * + * @param compressionLevel + * {@code 1} yields the fastest compression and {@code 9} yields the + * best compression. {@code 0} means no compression. The default + * compression level is {@code 6}. + * @param windowBits + * The base two logarithm of the size of the history buffer. The + * value should be in the range {@code 9} to {@code 15} inclusive. + * Larger values result in better compression at the expense of + * memory usage. The default value is {@code 15}. + * @param memLevel + * How much memory should be allocated for the internal compression + * state. {@code 1} uses minimum memory and {@code 9} uses maximum + * memory. Larger values result in better and faster compression + * at the expense of memory usage. The default value is {@code 8} + * + * @throws CompressionException if failed to initialize zlib + */ + public ZlibEncoder(ZlibWrapper wrapper, int compressionLevel, int windowBits, int memLevel) { if (compressionLevel < 0 || compressionLevel > 9) { throw new IllegalArgumentException( - "compressionLevel: " + compressionLevel + - " (expected: 0-9)"); + "compressionLevel: " + compressionLevel + " (expected: 0-9)"); + } + if (windowBits < 9 || windowBits > 15) { + throw new IllegalArgumentException( + "windowBits: " + windowBits + " (expected: 9-15)"); + } + if (memLevel < 1 || memLevel > 9) { + throw new IllegalArgumentException( + "memLevel: " + memLevel + " (expected: 1-9)"); } if (wrapper == null) { throw new NullPointerException("wrapper"); @@ -106,7 +144,8 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne } synchronized (z) { - int resultCode = z.deflateInit(compressionLevel, ZlibUtil.convertWrapperType(wrapper)); + int resultCode = z.deflateInit(compressionLevel, windowBits, memLevel, + ZlibUtil.convertWrapperType(wrapper)); if (resultCode != JZlib.Z_OK) { ZlibUtil.fail(z, "initialization failure", resultCode); } @@ -114,7 +153,8 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne } /** - * Creates a new zlib encoder with the default compression level ({@code 6}) + * Creates a new zlib encoder with the default compression level ({@code 6}), + * default window bits ({@code 15}), default memory level ({@code 8}), * and the specified preset dictionary. The wrapper is always * {@link ZlibWrapper#ZLIB} because it is the only format that supports * the preset dictionary. @@ -128,7 +168,8 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne } /** - * Creates a new zlib encoder with the specified {@code compressionLevel} + * Creates a new zlib encoder with the specified {@code compressionLevel}, + * default window bits ({@code 15}), default memory level ({@code 8}), * and the specified preset dictionary. The wrapper is always * {@link ZlibWrapper#ZLIB} because it is the only format that supports * the preset dictionary. @@ -142,17 +183,55 @@ public class ZlibEncoder extends OneToOneEncoder implements LifeCycleAwareChanne * @throws CompressionException if failed to initialize zlib */ public ZlibEncoder(int compressionLevel, byte[] dictionary) { - if (compressionLevel < 0 || compressionLevel > 9) { - throw new IllegalArgumentException("compressionLevel: " + compressionLevel + " (expected: 0-9)"); - } + this(compressionLevel, 15, 8, dictionary); + } + /** + * Creates a new zlib encoder with the specified {@code compressionLevel}, + * the specified {@code windowBits}, the specified {@code memLevel}, + * and the specified preset dictionary. The wrapper is always + * {@link ZlibWrapper#ZLIB} because it is the only format that supports + * the preset dictionary. + * + * @param compressionLevel + * {@code 1} yields the fastest compression and {@code 9} yields the + * best compression. {@code 0} means no compression. The default + * compression level is {@code 6}. + * @param windowBits + * The base two logarithm of the size of the history buffer. The + * value should be in the range {@code 9} to {@code 15} inclusive. + * Larger values result in better compression at the expense of + * memory usage. The default value is {@code 15}. + * @param memLevel + * How much memory should be allocated for the internal compression + * state. {@code 1} uses minimum memory and {@code 9} uses maximum + * memory. Larger values result in better and faster compression + * at the expense of memory usage. The default value is {@code 8} + * @param dictionary the preset dictionary + * + * @throws CompressionException if failed to initialize zlib + */ + public ZlibEncoder(int compressionLevel, int windowBits, int memLevel, byte[] dictionary) { + if (compressionLevel < 0 || compressionLevel > 9) { + throw new IllegalArgumentException( + "compressionLevel: " + compressionLevel + " (expected: 0-9)"); + } + if (windowBits < 9 || windowBits > 15) { + throw new IllegalArgumentException( + "windowBits: " + windowBits + " (expected: 9-15)"); + } + if (memLevel < 1 || memLevel > 9) { + throw new IllegalArgumentException( + "memLevel: " + memLevel + " (expected: 1-9)"); + } if (dictionary == null) { throw new NullPointerException("dictionary"); } synchronized (z) { int resultCode; - resultCode = z.deflateInit(compressionLevel, JZlib.W_ZLIB); // Default: ZLIB format + resultCode = z.deflateInit(compressionLevel, windowBits, memLevel, + JZlib.W_ZLIB); // Default: ZLIB format if (resultCode != JZlib.Z_OK) { ZlibUtil.fail(z, "initialization failure", resultCode); } else { diff --git a/src/main/java/org/jboss/netty/handler/codec/http/HttpContentCompressor.java b/src/main/java/org/jboss/netty/handler/codec/http/HttpContentCompressor.java index 803c2c53b5..3e85fed46f 100644 --- a/src/main/java/org/jboss/netty/handler/codec/http/HttpContentCompressor.java +++ b/src/main/java/org/jboss/netty/handler/codec/http/HttpContentCompressor.java @@ -30,16 +30,20 @@ import org.jboss.netty.handler.codec.embedder.EncoderEmbedder; public class HttpContentCompressor extends HttpContentEncoder { private final int compressionLevel; + private final int windowBits; + private final int memLevel; /** - * Creates a new handler with the default compression level (6). + * Creates a new handler with the default compression level (6), + * default window size (15) and default memory level (8). */ public HttpContentCompressor() { this(6); } /** - * Creates a new handler with the specified compression level. + * Creates a new handler with the specified compression level, default + * window size (15) and default memory level (8). * * @param compressionLevel * {@code 1} yields the fastest compression and {@code 9} yields the @@ -47,12 +51,44 @@ public class HttpContentCompressor extends HttpContentEncoder { * compression level is {@code 6}. */ public HttpContentCompressor(int compressionLevel) { + this(compressionLevel, 15, 8); + } + + /** + * Creates a new handler with the specified compression level, window size, + * and memory level.. + * + * @param compressionLevel + * {@code 1} yields the fastest compression and {@code 9} yields the + * best compression. {@code 0} means no compression. The default + * compression level is {@code 6}. + * @param windowBits + * The base two logarithm of the size of the history buffer. The + * value should be in the range {@code 9} to {@code 15} inclusive. + * Larger values result in better compression at the expense of + * memory usage. The default value is {@code 15}. + * @param memLevel + * How much memory should be allocated for the internal compression + * state. {@code 1} uses minimum memory and {@code 9} uses maximum + * memory. Larger values result in better and faster compression + * at the expense of memory usage. The default value is {@code 8} + */ + public HttpContentCompressor(int compressionLevel, int windowBits, int memLevel) { if (compressionLevel < 0 || compressionLevel > 9) { throw new IllegalArgumentException( - "compressionLevel: " + compressionLevel + - " (expected: 0-9)"); + "compressionLevel: " + compressionLevel + " (expected: 0-9)"); + } + if (windowBits < 9 || windowBits > 15) { + throw new IllegalArgumentException( + "windowBits: " + windowBits + " (expected: 9-15)"); + } + if (memLevel < 1 || memLevel > 9) { + throw new IllegalArgumentException( + "memLevel: " + memLevel + " (expected: 1-9)"); } this.compressionLevel = compressionLevel; + this.windowBits = windowBits; + this.memLevel = memLevel; } @Override @@ -62,7 +98,8 @@ public class HttpContentCompressor extends HttpContentEncoder { return null; } - return new EncoderEmbedder(new ZlibEncoder(wrapper, compressionLevel)); + return new EncoderEmbedder( + new ZlibEncoder(wrapper, compressionLevel, windowBits, memLevel)); } @Override diff --git a/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java b/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java index e98622109c..ba8fefcc28 100644 --- a/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java +++ b/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java @@ -1298,9 +1298,9 @@ final class Deflate { return lookahead; } - int deflateInit(ZStream strm, int level, int bits, WrapperType wrapperType) { + int deflateInit(ZStream strm, int level, int bits, int memLevel, WrapperType wrapperType) { return deflateInit2(strm, level, JZlib.Z_DEFLATED, bits, - JZlib.DEF_MEM_LEVEL, JZlib.Z_DEFAULT_STRATEGY, wrapperType); + memLevel, JZlib.Z_DEFAULT_STRATEGY, wrapperType); } private int deflateInit2(ZStream strm, int level, int method, int windowBits, diff --git a/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java b/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java index b28b595080..2fbd6c0fda 100644 --- a/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java +++ b/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java @@ -124,9 +124,13 @@ public final class ZStream { return deflateInit(level, bits, WrapperType.ZLIB); } - public int deflateInit(int level, int bits, @SuppressWarnings("rawtypes") Enum wrapperType) { + public int deflateInit(int level, int bits, Enum wrapperType) { + return deflateInit(level, bits, JZlib.DEF_MEM_LEVEL, wrapperType); + } + + public int deflateInit(int level, int bits, int memLevel, @SuppressWarnings("rawtypes") Enum wrapperType) { dstate = new Deflate(); - return dstate.deflateInit(this, level, bits, (WrapperType) wrapperType); + return dstate.deflateInit(this, level, bits, memLevel, (WrapperType) wrapperType); } public int deflate(int flush) {