Fixes for compression codecs
Motivation: Fixed founded mistakes in compression codecs. Modifications: - Changed return type of ZlibUtil.inflaterException() from CompressionException to DecompressionException - Updated @throws in javadoc of JZlibDecoder to throw DecompressionException instead of CompressionException - Fixed JdkZlibDecoder to throw DecompressionException instead of CompressionException - Removed unnecessary empty lines in JdkZlibEncoder and JZlibEncoder - Removed public modifier from Snappy class - Added MAX_UNCOMPRESSED_DATA_SIZE constant in SnappyFramedDecoder - Used in.readableBytes() instead of (in.writerIndex() - in.readerIndex()) in SnappyFramedDecoder - Added private modifier for enum ChunkType in SnappyFramedDecoder - Fixed potential bug (sum overflow) in Bzip2HuffmanAllocator.first(). For more info, see http://googleresearch.blogspot.ru/2006/06/extra-extra-read-all-about-it-nearly.html Result: Fixed sum overflow in Bzip2HuffmanAllocator, improved exceptions in ZlibDecoder implementations, hid Snappy class
This commit is contained in:
parent
ad1389be9d
commit
dc9d933d74
@ -42,7 +42,7 @@ final class Bzip2HuffmanAllocator {
|
|||||||
i = Math.max(nodesToMove - 1, i);
|
i = Math.max(nodesToMove - 1, i);
|
||||||
|
|
||||||
while (k > i + 1) {
|
while (k > i + 1) {
|
||||||
int temp = i + k >> 1;
|
int temp = i + k >>> 1;
|
||||||
if (array[temp] % length > limit) {
|
if (array[temp] % length > limit) {
|
||||||
k = temp;
|
k = temp;
|
||||||
} else {
|
} else {
|
||||||
|
@ -31,7 +31,7 @@ public class JZlibDecoder extends ZlibDecoder {
|
|||||||
/**
|
/**
|
||||||
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
|
* Creates a new instance with the default wrapper ({@link ZlibWrapper#ZLIB}).
|
||||||
*
|
*
|
||||||
* @throws CompressionException if failed to initialize zlib
|
* @throws DecompressionException if failed to initialize zlib
|
||||||
*/
|
*/
|
||||||
public JZlibDecoder() {
|
public JZlibDecoder() {
|
||||||
this(ZlibWrapper.ZLIB);
|
this(ZlibWrapper.ZLIB);
|
||||||
@ -40,7 +40,7 @@ public class JZlibDecoder extends ZlibDecoder {
|
|||||||
/**
|
/**
|
||||||
* Creates a new instance with the specified wrapper.
|
* Creates a new instance with the specified wrapper.
|
||||||
*
|
*
|
||||||
* @throws CompressionException if failed to initialize zlib
|
* @throws DecompressionException if failed to initialize zlib
|
||||||
*/
|
*/
|
||||||
public JZlibDecoder(ZlibWrapper wrapper) {
|
public JZlibDecoder(ZlibWrapper wrapper) {
|
||||||
if (wrapper == null) {
|
if (wrapper == null) {
|
||||||
@ -58,7 +58,7 @@ public class JZlibDecoder extends ZlibDecoder {
|
|||||||
* is always {@link ZlibWrapper#ZLIB} because it is the only format that
|
* is always {@link ZlibWrapper#ZLIB} because it is the only format that
|
||||||
* supports the preset dictionary.
|
* supports the preset dictionary.
|
||||||
*
|
*
|
||||||
* @throws CompressionException if failed to initialize zlib
|
* @throws DecompressionException if failed to initialize zlib
|
||||||
*/
|
*/
|
||||||
public JZlibDecoder(byte[] dictionary) {
|
public JZlibDecoder(byte[] dictionary) {
|
||||||
if (dictionary == null) {
|
if (dictionary == null) {
|
||||||
|
@ -29,7 +29,6 @@ import io.netty.util.internal.EmptyArrays;
|
|||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compresses a {@link ByteBuf} using the deflate algorithm.
|
* Compresses a {@link ByteBuf} using the deflate algorithm.
|
||||||
*/
|
*/
|
||||||
|
@ -24,7 +24,6 @@ import java.util.zip.DataFormatException;
|
|||||||
import java.util.zip.Deflater;
|
import java.util.zip.Deflater;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.Inflater;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Decompress a {@link ByteBuf} using the inflate algorithm.
|
* Decompress a {@link ByteBuf} using the inflate algorithm.
|
||||||
*/
|
*/
|
||||||
@ -249,14 +248,14 @@ public class JdkZlibDecoder extends ZlibDecoder {
|
|||||||
int magic1 = in.readByte();
|
int magic1 = in.readByte();
|
||||||
|
|
||||||
if (magic0 != 31) {
|
if (magic0 != 31) {
|
||||||
throw new CompressionException("Input is not in the GZIP format");
|
throw new DecompressionException("Input is not in the GZIP format");
|
||||||
}
|
}
|
||||||
crc.update(magic0);
|
crc.update(magic0);
|
||||||
crc.update(magic1);
|
crc.update(magic1);
|
||||||
|
|
||||||
int method = in.readUnsignedByte();
|
int method = in.readUnsignedByte();
|
||||||
if (method != Deflater.DEFLATED) {
|
if (method != Deflater.DEFLATED) {
|
||||||
throw new CompressionException("Unsupported compression method "
|
throw new DecompressionException("Unsupported compression method "
|
||||||
+ method + " in the GZIP header");
|
+ method + " in the GZIP header");
|
||||||
}
|
}
|
||||||
crc.update(method);
|
crc.update(method);
|
||||||
@ -265,7 +264,7 @@ public class JdkZlibDecoder extends ZlibDecoder {
|
|||||||
crc.update(flags);
|
crc.update(flags);
|
||||||
|
|
||||||
if ((flags & FRESERVED) != 0) {
|
if ((flags & FRESERVED) != 0) {
|
||||||
throw new CompressionException(
|
throw new DecompressionException(
|
||||||
"Reserved flags are set in the GZIP header");
|
"Reserved flags are set in the GZIP header");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -360,7 +359,7 @@ public class JdkZlibDecoder extends ZlibDecoder {
|
|||||||
}
|
}
|
||||||
int readLength = inflater.getTotalOut();
|
int readLength = inflater.getTotalOut();
|
||||||
if (dataLength != readLength) {
|
if (dataLength != readLength) {
|
||||||
throw new CompressionException(
|
throw new DecompressionException(
|
||||||
"Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
|
"Number of bytes mismatch. Expected: " + dataLength + ", Got: " + readLength);
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@ -373,7 +372,7 @@ public class JdkZlibDecoder extends ZlibDecoder {
|
|||||||
}
|
}
|
||||||
long readCrc = crc.getValue();
|
long readCrc = crc.getValue();
|
||||||
if (crcValue != readCrc) {
|
if (crcValue != readCrc) {
|
||||||
throw new CompressionException(
|
throw new DecompressionException(
|
||||||
"CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc);
|
"CRC value missmatch. Expected: " + crcValue + ", Got: " + readCrc);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -27,7 +27,6 @@ import java.util.concurrent.TimeUnit;
|
|||||||
import java.util.zip.CRC32;
|
import java.util.zip.CRC32;
|
||||||
import java.util.zip.Deflater;
|
import java.util.zip.Deflater;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compresses a {@link ByteBuf} using the deflate algorithm.
|
* Compresses a {@link ByteBuf} using the deflate algorithm.
|
||||||
*/
|
*/
|
||||||
|
@ -24,7 +24,7 @@ import io.netty.buffer.ByteBufUtil;
|
|||||||
*
|
*
|
||||||
* See http://code.google.com/p/snappy/source/browse/trunk/format_description.txt
|
* See http://code.google.com/p/snappy/source/browse/trunk/format_description.txt
|
||||||
*/
|
*/
|
||||||
public class Snappy {
|
class Snappy {
|
||||||
|
|
||||||
private static final int MAX_HT_SIZE = 1 << 14;
|
private static final int MAX_HT_SIZE = 1 << 14;
|
||||||
private static final int MIN_COMPRESSIBLE_BYTES = 15;
|
private static final int MIN_COMPRESSIBLE_BYTES = 15;
|
||||||
|
@ -36,7 +36,8 @@ import java.util.List;
|
|||||||
* set to {@code true}.
|
* set to {@code true}.
|
||||||
*/
|
*/
|
||||||
public class SnappyFramedDecoder extends ByteToMessageDecoder {
|
public class SnappyFramedDecoder extends ByteToMessageDecoder {
|
||||||
enum ChunkType {
|
|
||||||
|
private enum ChunkType {
|
||||||
STREAM_IDENTIFIER,
|
STREAM_IDENTIFIER,
|
||||||
COMPRESSED_DATA,
|
COMPRESSED_DATA,
|
||||||
UNCOMPRESSED_DATA,
|
UNCOMPRESSED_DATA,
|
||||||
@ -45,6 +46,7 @@ public class SnappyFramedDecoder extends ByteToMessageDecoder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static final byte[] SNAPPY = { 's', 'N', 'a', 'P', 'p', 'Y' };
|
private static final byte[] SNAPPY = { 's', 'N', 'a', 'P', 'p', 'Y' };
|
||||||
|
private static final int MAX_UNCOMPRESSED_DATA_SIZE = 65536 + 4;
|
||||||
|
|
||||||
private final Snappy snappy = new Snappy();
|
private final Snappy snappy = new Snappy();
|
||||||
private final boolean validateChecksums;
|
private final boolean validateChecksums;
|
||||||
@ -83,7 +85,7 @@ public class SnappyFramedDecoder extends ByteToMessageDecoder {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
int idx = in.readerIndex();
|
int idx = in.readerIndex();
|
||||||
final int inSize = in.writerIndex() - idx;
|
final int inSize = in.readableBytes();
|
||||||
if (inSize < 4) {
|
if (inSize < 4) {
|
||||||
// We need to be at least able to read the chunk type identifier (one byte),
|
// We need to be at least able to read the chunk type identifier (one byte),
|
||||||
// and the length of the chunk (3 bytes) in order to proceed
|
// and the length of the chunk (3 bytes) in order to proceed
|
||||||
@ -136,7 +138,7 @@ public class SnappyFramedDecoder extends ByteToMessageDecoder {
|
|||||||
if (!started) {
|
if (!started) {
|
||||||
throw new DecompressionException("Received UNCOMPRESSED_DATA tag before STREAM_IDENTIFIER");
|
throw new DecompressionException("Received UNCOMPRESSED_DATA tag before STREAM_IDENTIFIER");
|
||||||
}
|
}
|
||||||
if (chunkLength > 65536 + 4) {
|
if (chunkLength > MAX_UNCOMPRESSED_DATA_SIZE) {
|
||||||
throw new DecompressionException("Received UNCOMPRESSED_DATA larger than 65540 bytes");
|
throw new DecompressionException("Received UNCOMPRESSED_DATA larger than 65540 bytes");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -193,7 +195,7 @@ public class SnappyFramedDecoder extends ByteToMessageDecoder {
|
|||||||
* @param type The tag byte extracted from the stream
|
* @param type The tag byte extracted from the stream
|
||||||
* @return The appropriate {@link ChunkType}, defaulting to {@link ChunkType#RESERVED_UNSKIPPABLE}
|
* @return The appropriate {@link ChunkType}, defaulting to {@link ChunkType#RESERVED_UNSKIPPABLE}
|
||||||
*/
|
*/
|
||||||
static ChunkType mapChunkType(byte type) {
|
private static ChunkType mapChunkType(byte type) {
|
||||||
if (type == 0) {
|
if (type == 0) {
|
||||||
return ChunkType.COMPRESSED_DATA;
|
return ChunkType.COMPRESSED_DATA;
|
||||||
} else if (type == 1) {
|
} else if (type == 1) {
|
||||||
|
@ -32,8 +32,8 @@ final class ZlibUtil {
|
|||||||
throw deflaterException(z, message, resultCode);
|
throw deflaterException(z, message, resultCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
static CompressionException inflaterException(Inflater z, String message, int resultCode) {
|
static DecompressionException inflaterException(Inflater z, String message, int resultCode) {
|
||||||
return new CompressionException(message + " (" + resultCode + ')' + (z.msg != null? ": " + z.msg : ""));
|
return new DecompressionException(message + " (" + resultCode + ')' + (z.msg != null? ": " + z.msg : ""));
|
||||||
}
|
}
|
||||||
|
|
||||||
static CompressionException deflaterException(Deflater z, String message, int resultCode) {
|
static CompressionException deflaterException(Deflater z, String message, int resultCode) {
|
||||||
|
Loading…
Reference in New Issue
Block a user