andreacavalli/netty5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve Bzip2BitReader/Writer

Browse Source 
Motivation:

Before this changes Bzip2BitReader and Bzip2BitWriter accessed to ByteBuf byte by byte. So tests for Bzip2 compression codec takes a lot of time if we ran them with paranoid level of resource leak detection. For more information see comments to #2681 and #2689.

Modifications:

- Increased size of bit buffers from 8 to 64 bits.
- Improved reading and writing operations.
- Save link to incoming ByteBuf inside Bzip2BitReader.
- Added methods to check possible readable bits and bytes in Bzip2BitReader.
- Updated Bzip2 classes to use new API of Bzip2BitReader.
- Added new constants to Bzip2Constants.

Result:

Increased size of bit buffers and improved performance of Bzip2 compression codec (for general work by 13% and for tests with paranoid level of resource leak detection by 55%).
This commit is contained in:
Idel Pivnitskiy 2014-07-25 23:29:35 +04:00 committed by Norman Maurer
parent ff9cc74bf6
commit c13419750d
8 changed files with 195 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
codec/src/main/java/io/netty/handler/codec/compression/Bzip2BitReader.java
View File

@ -19,14 +19,24 @@ import io.netty.buffer.ByteBuf;
/** /**
* An bit reader that allows the reading of single bit booleans, bit strings of * An bit reader that allows the reading of single bit booleans, bit strings of
* arbitrary length (up to 24 bits), and bit aligned 32-bit integers. A single byte * arbitrary length (up to 32 bits), and bit aligned 32-bit integers. A single byte
* at a time is read from the {@link ByteBuf} when more bits are required. * at a time is read from the {@link ByteBuf} when more bits are required.
*/ */
class Bzip2BitReader { class Bzip2BitReader {
/**
* Maximum count of possible readable bytes to check.
*/
private static final int MAX_COUNT_OF_READABLE_BYTES = Integer.MAX_VALUE >>> 3;
/**
* The {@link ByteBuf} from which to read data.
*/
private ByteBuf in;
/** /**
* A buffer of bits read from the input stream that have not yet been returned. * A buffer of bits read from the input stream that have not yet been returned.
*/ */
private int bitBuffer; private long bitBuffer;
/** /**
* The number of bits currently buffered in {@link #bitBuffer}. * The number of bits currently buffered in {@link #bitBuffer}.
@ -34,53 +44,114 @@ class Bzip2BitReader {
private int bitCount; private int bitCount;
/** /**
* Reads up to 24 bits from the {@link ByteBuf}. * Set the {@link ByteBuf} from which to read data.
* @param count The number of bits to read (maximum {@code 24}, because the {@link #bitBuffer} */
* is {@code int} and it can store up to {@code 8} bits before calling) void setByteBuf(ByteBuf in) {
this.in = in;
}
/**
* Reads up to 32 bits from the {@link ByteBuf}.
* @param count The number of bits to read (maximum {@code 32} as a size of {@code int})
* @return The bits requested, right-aligned within the integer * @return The bits requested, right-aligned within the integer
*/ */
int readBits(ByteBuf in, final int count) { int readBits(final int count) {
if (count < 0 || count > 24) { if (count < 0 || count > 32) {
throw new IllegalArgumentException("count: " + count + " (expected: 0-24)"); throw new IllegalArgumentException("count: " + count + " (expected: 0-32 )");
} }
int bitCount = this.bitCount; int bitCount = this.bitCount;
int bitBuffer = this.bitBuffer; long bitBuffer = this.bitBuffer;
if (bitCount < count) { if (bitCount < count) {
do { long readData;
int uByte = in.readUnsignedByte(); int offset;
bitBuffer = bitBuffer << 8 | uByte; switch (in.readableBytes()) {
bitCount += 8; case 1: {
} while (bitCount < count); readData = in.readUnsignedByte();
offset = 8;
break;
}
case 2: {
readData = in.readUnsignedShort();
offset = 16;
break;
}
case 3: {
readData = in.readUnsignedMedium();
offset = 24;
break;
}
default: {
readData = in.readUnsignedInt();
offset = 32;
break;
}
}
bitBuffer = bitBuffer << offset | readData;
bitCount += offset;
this.bitBuffer = bitBuffer; this.bitBuffer = bitBuffer;
} }
this.bitCount = bitCount -= count; this.bitCount = bitCount -= count;
return (bitBuffer >>> bitCount) & ((1 << count) - 1); return (int) (bitBuffer >>> bitCount & (count != 32 ? (1 << count) - 1 : 0xFFFFFFFFL));
} }
/** /**
* Reads a single bit from the {@link ByteBuf}. * Reads a single bit from the {@link ByteBuf}.
* @return {@code true} if the bit read was {@code 1}, otherwise {@code false} * @return {@code true} if the bit read was {@code 1}, otherwise {@code false}
*/ */
boolean readBoolean(ByteBuf in) { boolean readBoolean() {
return readBits(in, 1) != 0; return readBits(1) != 0;
} }
/** /**
* Reads 32 bits of input as an integer. * Reads 32 bits of input as an integer.
* @return The integer read * @return The integer read
*/ */
int readInt(ByteBuf in) { int readInt() {
return readBits(in, 16) << 16 | readBits(in, 16); return readBits(32);
}
/**
* Refill the {@link ByteBuf} by one byte.
*/
void refill() {
int readData = in.readUnsignedByte();
bitBuffer = bitBuffer << 8 | readData;
bitCount += 8;
} }
/** /**
* Checks that at least one bit is available for reading. * Checks that at least one bit is available for reading.
* @return {@code true} if one bit is available for reading, otherwise {@code false} * @return {@code true} if one bit is available for reading, otherwise {@code false}
*/ */
boolean hasBit(ByteBuf in) { boolean isReadable() {
return bitCount > 0 || in.isReadable(); return bitCount > 0 || in.isReadable();
} }
/**
* Checks that the specified number of bits available for reading.
* @param count The number of bits to check
* @return {@code true} if {@code count} bits are available for reading, otherwise {@code false}
*/
boolean hasReadableBits(int count) {
if (count < 0) {
throw new IllegalArgumentException("count: " + count + " (expected value greater than 0)");
}
return bitCount >= count || (in.readableBytes() << 3 & Integer.MAX_VALUE) >= count - bitCount;
}
/**
* Checks that the specified number of bytes available for reading.
* @param count The number of bytes to check
* @return {@code true} if {@code count} bytes are available for reading, otherwise {@code false}
*/
boolean hasReadableBytes(int count) {
if (count < 0 || count > MAX_COUNT_OF_READABLE_BYTES) {
throw new IllegalArgumentException("count: " + count
+ " (expected: 0-" + MAX_COUNT_OF_READABLE_BYTES + ')');
}
return hasReadableBits(count << 3);
}
} }

49
codec/src/main/java/io/netty/handler/codec/compression/Bzip2BitWriter.java
View File

@ -19,14 +19,14 @@ import io.netty.buffer.ByteBuf;
/** /**
* A bit writer that allows the writing of single bit booleans, unary numbers, bit strings * A bit writer that allows the writing of single bit booleans, unary numbers, bit strings
* of arbitrary length (up to 24 bits), and bit aligned 32-bit integers. A single byte at a * of arbitrary length (up to 32 bits), and bit aligned 32-bit integers. A single byte at a
* time is written to the {@link ByteBuf} when sufficient bits have been accumulated. * time is written to the {@link ByteBuf} when sufficient bits have been accumulated.
*/ */
final class Bzip2BitWriter { final class Bzip2BitWriter {
/** /**
* A buffer of bits waiting to be written to the output stream. * A buffer of bits waiting to be written to the output stream.
*/ */
private int bitBuffer; private long bitBuffer;
/** /**
* The number of bits currently buffered in {@link #bitBuffer}. * The number of bits currently buffered in {@link #bitBuffer}.
@ -34,23 +34,22 @@ final class Bzip2BitWriter {
private int bitCount; private int bitCount;
/** /**
* Writes up to 24 bits to the output {@link ByteBuf}. * Writes up to 32 bits to the output {@link ByteBuf}.
* @param count The number of bits to write (maximum {@code 24}, because the {@link #bitBuffer} * @param count The number of bits to write (maximum {@code 32} as a size of {@code int})
* is {@code int} and it can store up to {@code 8} bits before calling)
* @param value The bits to write * @param value The bits to write
*/ */
void writeBits(ByteBuf out, final int count, final int value) { void writeBits(ByteBuf out, final int count, final long value) {
if (count < 0 || count > 24) { if (count < 0 || count > 32) {
throw new IllegalArgumentException("count: " + count + " (expected: 0-24)"); throw new IllegalArgumentException("count: " + count + " (expected: 0-32)");
} }
int bitCount = this.bitCount; int bitCount = this.bitCount;
int bitBuffer = this.bitBuffer | (value << (32 - count)) >>> bitCount; long bitBuffer = this.bitBuffer | value << 64 - count >>> bitCount;
bitCount += count; bitCount += count;
while (bitCount >= 8) { if (bitCount >= 32) {
out.writeByte(bitBuffer >>> 24); out.writeInt((int) (bitBuffer >>> 32));
bitBuffer <<= 8; bitBuffer <<= 32;
bitCount -= 8; bitCount -= 32;
} }
this.bitBuffer = bitBuffer; this.bitBuffer = bitBuffer;
this.bitCount = bitCount; this.bitCount = bitCount;
@ -62,10 +61,10 @@ final class Bzip2BitWriter {
*/ */
void writeBoolean(ByteBuf out, final boolean value) { void writeBoolean(ByteBuf out, final boolean value) {
int bitCount = this.bitCount + 1; int bitCount = this.bitCount + 1;
int bitBuffer = this.bitBuffer | (value ? 1 : 0) << (32 - bitCount); long bitBuffer = this.bitBuffer | (value ? 1L << 64 - bitCount : 0L);
if (bitCount == 8) { if (bitCount == 32) {
out.writeByte(bitBuffer >>> 24); out.writeInt((int) (bitBuffer >>> 32));
bitBuffer = 0; bitBuffer = 0;
bitCount = 0; bitCount = 0;
} }
@ -93,8 +92,7 @@ final class Bzip2BitWriter {
* @param value The integer to write * @param value The integer to write
*/ */
void writeInt(ByteBuf out, final int value) { void writeInt(ByteBuf out, final int value) {
writeBits(out, 16, (value >>> 16) & 0xffff); writeBits(out, 32, value);
writeBits(out, 16, value & 0xffff);
} }
/** /**
@ -102,8 +100,21 @@ final class Bzip2BitWriter {
* zero padding to a whole byte as required. * zero padding to a whole byte as required.
*/ */
void flush(ByteBuf out) { void flush(ByteBuf out) {
final int bitCount = this.bitCount;
if (bitCount > 0) { if (bitCount > 0) {
writeBits(out, 8 - bitCount, 0); final long bitBuffer = this.bitBuffer;
final int shiftToRight = 64 - bitCount;
if (bitCount <= 8) {
out.writeByte((int) (bitBuffer >>> shiftToRight << 8 - bitCount));
} else if (bitCount <= 16) {
out.writeShort((int) (bitBuffer >>> shiftToRight << 16 - bitCount));
} else if (bitCount <= 24) {
out.writeMedium((int) (bitBuffer >>> shiftToRight << 24 - bitCount));
} else {
out.writeInt((int) (bitBuffer >>> shiftToRight << 32 - bitCount));
}
} }
} }
} }

4
codec/src/main/java/io/netty/handler/codec/compression/Bzip2BlockCompressor.java
View File

@ -102,7 +102,7 @@ final class Bzip2BlockCompressor {
final boolean[] condensedInUse = new boolean[16]; final boolean[] condensedInUse = new boolean[16];
for (int i = 0; i < condensedInUse.length; i++) { for (int i = 0; i < condensedInUse.length; i++) {
for (int j = 0, k = i << 4; j < 16; j++, k++) { for (int j = 0, k = i << 4; j < HUFFMAN_SYMBOL_RANGE_SIZE; j++, k++) {
if (blockValuesPresent[k]) { if (blockValuesPresent[k]) {
condensedInUse[i] = true; condensedInUse[i] = true;
} }
@ -115,7 +115,7 @@ final class Bzip2BlockCompressor {
for (int i = 0; i < condensedInUse.length; i++) { for (int i = 0; i < condensedInUse.length; i++) {
if (condensedInUse[i]) { if (condensedInUse[i]) {
for (int j = 0, k = i << 4; j < 16; j++, k++) { for (int j = 0, k = i << 4; j < HUFFMAN_SYMBOL_RANGE_SIZE; j++, k++) {
writer.writeBoolean(out, blockValuesPresent[k]); writer.writeBoolean(out, blockValuesPresent[k]);
} }
} }

35
codec/src/main/java/io/netty/handler/codec/compression/Bzip2BlockDecompressor.java
View File

@ -15,8 +15,6 @@
*/ */
package io.netty.handler.codec.compression; package io.netty.handler.codec.compression;
import io.netty.buffer.ByteBuf;
import static io.netty.handler.codec.compression.Bzip2Constants.*; import static io.netty.handler.codec.compression.Bzip2Constants.*;
/** /**
@ -25,14 +23,19 @@ import static io.netty.handler.codec.compression.Bzip2Constants.*;
* Block decoding consists of the following stages:<br> * Block decoding consists of the following stages:<br>
* 1. Read block header<br> * 1. Read block header<br>
* 2. Read Huffman tables<br> * 2. Read Huffman tables<br>
* 3. Read and decode Huffman encoded data - {@link #decodeHuffmanData(Bzip2HuffmanStageDecoder, ByteBuf)}<br> * 3. Read and decode Huffman encoded data - {@link #decodeHuffmanData(Bzip2HuffmanStageDecoder)}<br>
* 4. Run-Length Decoding[2] - {@link #decodeHuffmanData(Bzip2HuffmanStageDecoder, ByteBuf)}<br> * 4. Run-Length Decoding[2] - {@link #decodeHuffmanData(Bzip2HuffmanStageDecoder)}<br>
* 5. Inverse Move To Front Transform - {@link #decodeHuffmanData(Bzip2HuffmanStageDecoder, ByteBuf)}<br> * 5. Inverse Move To Front Transform - {@link #decodeHuffmanData(Bzip2HuffmanStageDecoder)}<br>
* 6. Inverse Burrows Wheeler Transform - {@link #initialiseInverseBWT()}<br> * 6. Inverse Burrows Wheeler Transform - {@link #initialiseInverseBWT()}<br>
* 7. Run-Length Decoding[1] - {@link #read()}<br> * 7. Run-Length Decoding[1] - {@link #read()}<br>
* 8. Optional Block De-Randomisation - {@link #read()} (through {@link #decodeNextBWTByte()}) * 8. Optional Block De-Randomisation - {@link #read()} (through {@link #decodeNextBWTByte()})
*/ */
final class Bzip2BlockDecompressor { final class Bzip2BlockDecompressor {
/**
* A reader that provides bit-level reads.
*/
private final Bzip2BitReader reader;
/** /**
* Calculates the block CRC from the fully decoded bytes of the block. * Calculates the block CRC from the fully decoded bytes of the block.
*/ */
@ -142,25 +145,31 @@ final class Bzip2BlockDecompressor {
/** /**
* Table for Move To Front transformations. * Table for Move To Front transformations.
*/ */
final Bzip2MoveToFrontTable symbolMTF = new Bzip2MoveToFrontTable(); private final Bzip2MoveToFrontTable symbolMTF = new Bzip2MoveToFrontTable();
int repeatCount; // This variables is used to save current state if we haven't got enough readable bits
int repeatIncrement = 1; private int repeatCount;
int mtfValue; private int repeatIncrement = 1;
private int mtfValue;
Bzip2BlockDecompressor(final int blockSize, final int blockCRC, final boolean blockRandomised,
final int bwtStartPointer, final Bzip2BitReader reader) {
Bzip2BlockDecompressor(int blockSize, int blockCRC, boolean blockRandomised, int bwtStartPointer) {
bwtBlock = new byte[blockSize]; bwtBlock = new byte[blockSize];
this.blockCRC = blockCRC; this.blockCRC = blockCRC;
this.blockRandomised = blockRandomised; this.blockRandomised = blockRandomised;
this.bwtStartPointer = bwtStartPointer; this.bwtStartPointer = bwtStartPointer;
this.reader = reader;
} }
/** /**
* Reads the Huffman encoded data from the input stream, performs Run-Length Decoding and * Reads the Huffman encoded data from the input stream, performs Run-Length Decoding and
* applies the Move To Front transform to reconstruct the Burrows-Wheeler Transform array. * applies the Move To Front transform to reconstruct the Burrows-Wheeler Transform array.
*/ */
boolean decodeHuffmanData(final Bzip2HuffmanStageDecoder huffmanDecoder, ByteBuf in) { boolean decodeHuffmanData(final Bzip2HuffmanStageDecoder huffmanDecoder) {
final Bzip2BitReader reader = this.reader;
final byte[] bwtBlock = this.bwtBlock; final byte[] bwtBlock = this.bwtBlock;
final byte[] huffmanSymbolMap = this.huffmanSymbolMap; final byte[] huffmanSymbolMap = this.huffmanSymbolMap;
final int streamBlockSize = this.bwtBlock.length; final int streamBlockSize = this.bwtBlock.length;
@ -174,14 +183,14 @@ final class Bzip2BlockDecompressor {
int mtfValue = this.mtfValue; int mtfValue = this.mtfValue;
for (;;) { for (;;) {
if (in.readableBytes() < 3) { // 3 = (HUFFMAN_DECODE_MAX_CODE_LENGTH + 1) bits / 8 if (!reader.hasReadableBits(HUFFMAN_DECODE_MAX_CODE_LENGTH)) {
this.bwtBlockLength = bwtBlockLength; this.bwtBlockLength = bwtBlockLength;
this.repeatCount = repeatCount; this.repeatCount = repeatCount;
this.repeatIncrement = repeatIncrement; this.repeatIncrement = repeatIncrement;
this.mtfValue = mtfValue; this.mtfValue = mtfValue;
return false; return false;
} }
final int nextSymbol = huffmanDecoder.nextSymbol(in); final int nextSymbol = huffmanDecoder.nextSymbol();
if (nextSymbol == HUFFMAN_SYMBOL_RUNA) { if (nextSymbol == HUFFMAN_SYMBOL_RUNA) {
repeatCount += repeatIncrement; repeatCount += repeatIncrement;

10
codec/src/main/java/io/netty/handler/codec/compression/Bzip2Constants.java
View File

@ -70,6 +70,16 @@ final class Bzip2Constants {
static final int HUFFMAN_SYMBOL_RUNA = 0; static final int HUFFMAN_SYMBOL_RUNA = 0;
static final int HUFFMAN_SYMBOL_RUNB = 1; static final int HUFFMAN_SYMBOL_RUNB = 1;
/**
* Huffman symbols range size for Huffman used map.
*/
static final int HUFFMAN_SYMBOL_RANGE_SIZE = 16;
/**
* Maximum length of zero-terminated bit runs of MTF'ed Huffman table.
*/
static final int HUFFMAN_SELECTOR_LIST_MAX_LENGTH = 6;
/** /**
* Number of symbols decoded after which a new Huffman table is selected. * Number of symbols decoded after which a new Huffman table is selected.
*/ */

71
codec/src/main/java/io/netty/handler/codec/compression/Bzip2Decoder.java
View File

@ -42,7 +42,6 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
RECEIVE_SELECTORS, RECEIVE_SELECTORS,
RECEIVE_HUFFMAN_LENGTH, RECEIVE_HUFFMAN_LENGTH,
DECODE_HUFFMAN_DATA, DECODE_HUFFMAN_DATA,
END_BLOCK,
EOF EOF
} }
private State currentState = State.INIT; private State currentState = State.INIT;
@ -82,6 +81,8 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
if (!in.isReadable()) { if (!in.isReadable()) {
return; return;
} }
final Bzip2BitReader reader = this.reader;
reader.setByteBuf(in);
for (;;) { for (;;) {
switch (currentState) { switch (currentState) {
@ -103,16 +104,15 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
streamCRC = 0; streamCRC = 0;
currentState = State.INIT_BLOCK; currentState = State.INIT_BLOCK;
case INIT_BLOCK: case INIT_BLOCK:
if (in.readableBytes() < 10) { if (!reader.hasReadableBytes(10)) {
return; return;
} }
Bzip2BitReader reader = this.reader;
// Get the block magic bytes. // Get the block magic bytes.
final int magic1 = reader.readBits(in, 24); final int magic1 = reader.readBits(24);
final int magic2 = reader.readBits(in, 24); final int magic2 = reader.readBits(24);
if (magic1 == END_OF_STREAM_MAGIC_1 && magic2 == END_OF_STREAM_MAGIC_2) { if (magic1 == END_OF_STREAM_MAGIC_1 && magic2 == END_OF_STREAM_MAGIC_2) {
// End of stream was reached. Check the combined CRC. // End of stream was reached. Check the combined CRC.
final int storedCombinedCRC = reader.readInt(in); final int storedCombinedCRC = reader.readInt();
if (storedCombinedCRC != streamCRC) { if (storedCombinedCRC != streamCRC) {
throw new DecompressionException("stream CRC error"); throw new DecompressionException("stream CRC error");
} }
@ -122,25 +122,23 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
if (magic1 != BLOCK_HEADER_MAGIC_1 || magic2 != BLOCK_HEADER_MAGIC_2) { if (magic1 != BLOCK_HEADER_MAGIC_1 || magic2 != BLOCK_HEADER_MAGIC_2) {
throw new DecompressionException("bad block header"); throw new DecompressionException("bad block header");
} }
blockCRC = reader.readInt(in); blockCRC = reader.readInt();
currentState = State.INIT_BLOCK_PARAMS; currentState = State.INIT_BLOCK_PARAMS;
case INIT_BLOCK_PARAMS: case INIT_BLOCK_PARAMS:
if (in.readableBytes() < 4) { if (!reader.hasReadableBits(25)) {
return; return;
} }
reader = this.reader; final boolean blockRandomised = reader.readBoolean();
final boolean blockRandomised = reader.readBoolean(in); final int bwtStartPointer = reader.readBits(24);
final int bwtStartPointer = reader.readBits(in, 24);
blockDecompressor = new Bzip2BlockDecompressor(this.blockSize, blockCRC, blockDecompressor = new Bzip2BlockDecompressor(this.blockSize, blockCRC,
blockRandomised, bwtStartPointer); blockRandomised, bwtStartPointer, reader);
currentState = State.RECEIVE_HUFFMAN_USED_MAP; currentState = State.RECEIVE_HUFFMAN_USED_MAP;
case RECEIVE_HUFFMAN_USED_MAP: case RECEIVE_HUFFMAN_USED_MAP:
if (in.readableBytes() < 2) { if (!reader.hasReadableBits(16)) {
return; return;
} }
reader = this.reader; blockDecompressor.huffmanInUse16 = reader.readBits(16);
blockDecompressor.huffmanInUse16 = reader.readBits(in, 16);
currentState = State.RECEIVE_HUFFMAN_USED_BITMAPS; currentState = State.RECEIVE_HUFFMAN_USED_BITMAPS;
case RECEIVE_HUFFMAN_USED_BITMAPS: case RECEIVE_HUFFMAN_USED_BITMAPS:
Bzip2BlockDecompressor blockDecompressor = this.blockDecompressor; Bzip2BlockDecompressor blockDecompressor = this.blockDecompressor;
@ -148,17 +146,16 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
final int bitNumber = Integer.bitCount(inUse16); final int bitNumber = Integer.bitCount(inUse16);
final byte[] huffmanSymbolMap = blockDecompressor.huffmanSymbolMap; final byte[] huffmanSymbolMap = blockDecompressor.huffmanSymbolMap;
if (in.readableBytes() < bitNumber * 16 / 8 + 1) { if (!reader.hasReadableBits(bitNumber * HUFFMAN_SYMBOL_RANGE_SIZE + 3)) {
return; return;
} }
reader = this.reader;
int huffmanSymbolCount = 0; int huffmanSymbolCount = 0;
if (bitNumber > 0) { if (bitNumber > 0) {
for (int i = 0; i < 16; i++) { for (int i = 0; i < 16; i++) {
if ((inUse16 & 1 << 15 >>> i) != 0) { if ((inUse16 & 1 << 15 >>> i) != 0) {
for (int j = 0, k = i << 4; j < 16; j++, k++) { for (int j = 0, k = i << 4; j < HUFFMAN_SYMBOL_RANGE_SIZE; j++, k++) {
if (reader.readBoolean(in)) { if (reader.readBoolean()) {
huffmanSymbolMap[huffmanSymbolCount++] = (byte) k; huffmanSymbolMap[huffmanSymbolCount++] = (byte) k;
} }
} }
@ -167,7 +164,7 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
} }
blockDecompressor.huffmanEndOfBlockSymbol = huffmanSymbolCount + 1; blockDecompressor.huffmanEndOfBlockSymbol = huffmanSymbolCount + 1;
int totalTables = reader.readBits(in, 3); int totalTables = reader.readBits(3);
if (totalTables < HUFFMAN_MINIMUM_TABLES || totalTables > HUFFMAN_MAXIMUM_TABLES) { if (totalTables < HUFFMAN_MINIMUM_TABLES || totalTables > HUFFMAN_MAXIMUM_TABLES) {
throw new DecompressionException("incorrect huffman groups number"); throw new DecompressionException("incorrect huffman groups number");
} }
@ -178,11 +175,10 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
huffmanStageDecoder = new Bzip2HuffmanStageDecoder(reader, totalTables, alphaSize); huffmanStageDecoder = new Bzip2HuffmanStageDecoder(reader, totalTables, alphaSize);
currentState = State.RECEIVE_SELECTORS_NUMBER; currentState = State.RECEIVE_SELECTORS_NUMBER;
case RECEIVE_SELECTORS_NUMBER: case RECEIVE_SELECTORS_NUMBER:
if (in.readableBytes() < 2) { if (!reader.hasReadableBits(15)) {
return; return;
} }
reader = this.reader; int totalSelectors = reader.readBits(15);
int totalSelectors = reader.readBits(in, 15);
if (totalSelectors < 1 || totalSelectors > MAX_SELECTORS) { if (totalSelectors < 1 || totalSelectors > MAX_SELECTORS) {
throw new DecompressionException("incorrect selectors number"); throw new DecompressionException("incorrect selectors number");
} }
@ -194,19 +190,18 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
byte[] selectors = huffmanStageDecoder.selectors; byte[] selectors = huffmanStageDecoder.selectors;
totalSelectors = selectors.length; totalSelectors = selectors.length;
final Bzip2MoveToFrontTable tableMtf = huffmanStageDecoder.tableMTF; final Bzip2MoveToFrontTable tableMtf = huffmanStageDecoder.tableMTF;
reader = this.reader;
int currSelector; int currSelector;
// Get zero-terminated bit runs (0..62) of MTF'ed Huffman table. length = 1..6 // Get zero-terminated bit runs (0..62) of MTF'ed Huffman table. length = 1..6
for (currSelector = huffmanStageDecoder.currentSelector; for (currSelector = huffmanStageDecoder.currentSelector;
currSelector < totalSelectors; currSelector++) { currSelector < totalSelectors; currSelector++) {
if (!in.isReadable()) { if (!reader.hasReadableBits(HUFFMAN_SELECTOR_LIST_MAX_LENGTH)) {
// Save state if end of current ByteBuf was reached // Save state if end of current ByteBuf was reached
huffmanStageDecoder.currentSelector = currSelector; huffmanStageDecoder.currentSelector = currSelector;
return; return;
} }
int index = 0; int index = 0;
while (reader.readBoolean(in)) { while (reader.readBoolean()) {
index++; index++;
} }
selectors[currSelector] = tableMtf.indexToFront(index); selectors[currSelector] = tableMtf.indexToFront(index);
@ -218,7 +213,6 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
totalTables = huffmanStageDecoder.totalTables; totalTables = huffmanStageDecoder.totalTables;
final byte[][] codeLength = huffmanStageDecoder.tableCodeLengths; final byte[][] codeLength = huffmanStageDecoder.tableCodeLengths;
alphaSize = huffmanStageDecoder.alphabetSize; alphaSize = huffmanStageDecoder.alphabetSize;
reader = this.reader;
/* Now the coding tables */ /* Now the coding tables */
int currGroup; int currGroup;
@ -228,29 +222,29 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
boolean saveStateAndReturn = false; boolean saveStateAndReturn = false;
loop: for (currGroup = huffmanStageDecoder.currentGroup; currGroup < totalTables; currGroup++) { loop: for (currGroup = huffmanStageDecoder.currentGroup; currGroup < totalTables; currGroup++) {
// start_huffman_length // start_huffman_length
if (!in.isReadable()) { if (!reader.hasReadableBits(5)) {
saveStateAndReturn = true; saveStateAndReturn = true;
break; break;
} }
if (currLength < 0) { if (currLength < 0) {
currLength = reader.readBits(in, 5); currLength = reader.readBits(5);
} }
for (currAlpha = huffmanStageDecoder.currentAlpha; currAlpha < alphaSize; currAlpha++) { for (currAlpha = huffmanStageDecoder.currentAlpha; currAlpha < alphaSize; currAlpha++) {
// delta_bit_length: 1..40 // delta_bit_length: 1..40
if (!reader.hasBit(in)) { if (!reader.isReadable()) {
saveStateAndReturn = true; saveStateAndReturn = true;
break loop; break loop;
} }
while (modifyLength || reader.readBoolean(in)) { // 0=>next symbol; 1=>alter length while (modifyLength || reader.readBoolean()) { // 0=>next symbol; 1=>alter length
if (!reader.hasBit(in)) { if (!reader.isReadable()) {
modifyLength = true; modifyLength = true;
saveStateAndReturn = true; saveStateAndReturn = true;
break loop; break loop;
} }
// 1=>decrement length; 0=>increment length // 1=>decrement length; 0=>increment length
currLength += reader.readBoolean(in) ? -1 : 1; currLength += reader.readBoolean() ? -1 : 1;
modifyLength = false; modifyLength = false;
if (!reader.hasBit(in)) { if (!reader.isReadable()) {
saveStateAndReturn = true; saveStateAndReturn = true;
break loop; break loop;
} }
@ -275,10 +269,17 @@ public class Bzip2Decoder extends ByteToMessageDecoder {
currentState = State.DECODE_HUFFMAN_DATA; currentState = State.DECODE_HUFFMAN_DATA;
case DECODE_HUFFMAN_DATA: case DECODE_HUFFMAN_DATA:
blockDecompressor = this.blockDecompressor; blockDecompressor = this.blockDecompressor;
final boolean decoded = blockDecompressor.decodeHuffmanData(this.huffmanStageDecoder, in); final int oldReaderIndex = in.readerIndex();
final boolean decoded = blockDecompressor.decodeHuffmanData(this.huffmanStageDecoder);
if (!decoded) { if (!decoded) {
return; return;
} }
// It used to avoid "Bzip2Decoder.decode() did not read anything but decoded a message" exception.
// Because previous operation may read only a few bits from Bzip2BitReader.bitBuffer and
// don't read incomming ByteBuf.
if (in.readerIndex() == oldReaderIndex && in.isReadable()) {
reader.refill();
}
final int blockLength = blockDecompressor.blockLength(); final int blockLength = blockDecompressor.blockLength();
final ByteBuf uncompressed = ctx.alloc().buffer(blockLength); final ByteBuf uncompressed = ctx.alloc().buffer(blockLength);

3
codec/src/main/java/io/netty/handler/codec/compression/Bzip2Encoder.java
View File

@ -42,8 +42,7 @@ public class Bzip2Encoder extends MessageToByteEncoder<ByteBuf> {
INIT, INIT,
INIT_BLOCK, INIT_BLOCK,
WRITE_DATA, WRITE_DATA,
CLOSE_BLOCK, CLOSE_BLOCK
EOF
} }
private State currentState = State.INIT; private State currentState = State.INIT;

8
codec/src/main/java/io/netty/handler/codec/compression/Bzip2HuffmanStageDecoder.java
View File

@ -15,8 +15,6 @@
*/ */
package io.netty.handler.codec.compression; package io.netty.handler.codec.compression;
import io.netty.buffer.ByteBuf;
import static io.netty.handler.codec.compression.Bzip2Constants.*; import static io.netty.handler.codec.compression.Bzip2Constants.*;
/** /**
@ -170,7 +168,7 @@ final class Bzip2HuffmanStageDecoder {
* Decodes and returns the next symbol. * Decodes and returns the next symbol.
* @return The decoded symbol * @return The decoded symbol
*/ */
int nextSymbol(ByteBuf in) { int nextSymbol() {
// Move to next group selector if required // Move to next group selector if required
if (++groupPosition % HUFFMAN_GROUP_RUN_LENGTH == 0) { if (++groupPosition % HUFFMAN_GROUP_RUN_LENGTH == 0) {
groupIndex++; groupIndex++;
@ -189,13 +187,13 @@ final class Bzip2HuffmanStageDecoder {
// Starting with the minimum bit length for the table, read additional bits one at a time // Starting with the minimum bit length for the table, read additional bits one at a time
// until a complete code is recognised // until a complete code is recognised
int codeBits = reader.readBits(in, codeLength); int codeBits = reader.readBits(codeLength);
for (; codeLength <= HUFFMAN_DECODE_MAX_CODE_LENGTH; codeLength++) { for (; codeLength <= HUFFMAN_DECODE_MAX_CODE_LENGTH; codeLength++) {
if (codeBits <= tableLimits[codeLength]) { if (codeBits <= tableLimits[codeLength]) {
// Convert the code to a symbol index and return // Convert the code to a symbol index and return
return tableSymbols[codeBits - tableBases[codeLength]]; return tableSymbols[codeBits - tableBases[codeLength]];
} }
codeBits = codeBits << 1 | reader.readBits(in, 1); codeBits = codeBits << 1 | reader.readBits(1);
} }
throw new DecompressionException("a valid code was not recognised"); throw new DecompressionException("a valid code was not recognised");